diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2a3000ef8..b7817f12d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,8 @@ * @EtienneLescot + +# Linux / Wayland capture — @Beetix +/electron/native/pipewire-capture/ @Beetix @EtienneLescot +/electron/native-bridge/capture/linuxNativeCaptureSession* @Beetix @EtienneLescot +/electron/native-bridge/cursor/recording/pipeWire* @Beetix @EtienneLescot +/crates/compositor/src/linux_*.rs @Beetix @EtienneLescot +/crates/compositor/src/*_linux.rs @Beetix @EtienneLescot diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..993fd236d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +github: [EtienneLescot] +ko_fi: etiennelescot diff --git a/.github/workflows/build-onnxruntime-macos.yml b/.github/workflows/build-onnxruntime-macos.yml new file mode 100644 index 000000000..22aef9432 --- /dev/null +++ b/.github/workflows/build-onnxruntime-macos.yml @@ -0,0 +1,315 @@ +name: Build ONNX Runtime (macOS 13 floor) + +# Builds ONNX Runtime for macOS arm64 with the deployment target pinned to the +# floor this app declares, and publishes it as an artifact for a maintainer to +# attach to a release. +# +# WHY THIS EXISTS. Microsoft's own `onnxruntime-osx-arm64-*.tgz` is built for +# macOS 14. `electron-builder.json5` declares `minimumSystemVersion: "13.0"`, +# and `scripts/before-pack.cjs` refuses to package any binary that demands more +# than the floor — correctly, because the deployment target decides which +# symbols the linker resolves against the OS rather than emitting locally, so a +# too-high floor strands users on the older OS with `Symbol not found` at dyld +# time (#515). The result is that `npm run build:mac` cannot package at all. +# +# Every published release from 1.24 onward is `minos 14.0`, and every release +# before it is at least 13.3, so no prebuilt artifact has ever satisfied a 13.0 +# floor. Building it is the only way to keep both macOS 13 support and webcam +# segmentation. +# +# WHAT IS NOT BUILT. The CoreML execution provider. `segmentation.rs` builds its +# session with no explicit provider, i.e. the CPU EP, which +# technical-documentation/engineering/webcam-segmentation.md records as a +# measured decision ("Inference p50, CPU EP: 3.575 ms — the CPU is faster"). +# Dropping CoreML is what takes the library from 36.7 MB to ~22 MB. +# +# HOW IT IS PUBLISHED. This workflow does not create releases. It builds, +# verifies, and prints the exact `PINNED` entry for +# `scripts/fetch-onnxruntime.mjs` in the job summary; attaching the archive to a +# release and pasting that entry stays a deliberate human step, so the +# supply-chain posture the script documents — immutable URL, SHA-256 verified +# before the archive is opened — is preserved rather than replaced by "whatever +# CI last uploaded". + +on: + workflow_dispatch: + push: + # BRANCHES ONLY. Without this, pushing a tag matches too — and publishing the + # artifact under `v0.0.0-onnxruntime-1.27.1` started a fresh 22-minute build of + # the very thing that had just been attached to the release. + branches: + - main + paths: + - ".github/workflows/build-onnxruntime-macos.yml" + - "scripts/fetch-onnxruntime.mjs" + +# One build per branch. Without this, every push to a branch that touches the paths +# above starts another 1-2 h build and none of the earlier ones stop: three ran +# concurrently on this workflow's own PR, the oldest for nearly four hours. +concurrency: + group: onnxruntime-macos-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + # Provenance attestation. `id-token` mints the OIDC token GitHub signs with, and + # `attestations` lets the run record the result. Both are needed by + # `actions/attest-build-provenance`; neither grants write access to the repository. + id-token: write + attestations: write + +jobs: + build: + name: macOS arm64, deployment target 13.0 + # arm64 runner: the only macOS target upstream ships, and the only one the + # app packages. There is no Intel build to match. + runs-on: macos-latest + # The default is 6 h. This build takes ~10 min on an 8-core M1 with `--parallel 4` + # and well over an hour on the runner; a cap turns "pathologically slow" into a + # failure somebody sees rather than six hours of quietly burnt minutes. + timeout-minutes: 150 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # This job compiles third-party source in the same workspace. Leaving the + # token in .git/config would put it within reach of ONNX Runtime's own build + # scripts — and this workflow holds `attestations: write`. Same setting + # build.yml, docs.yml and nix-build.yml already use. + persist-credentials: false + + - name: Read the pinned version from fetch-onnxruntime.mjs + id: pin + # Single source of truth. `fetch-onnxruntime.test.mjs` already cross-checks + # that VERSION satisfies the `api-NN` feature `crates/Cargo.toml` gives + # `ort`; reading it here rather than repeating it means a bump cannot leave + # this workflow building a version nothing consumes. + run: | + set -euo pipefail + VERSION="$(sed -n 's/^const VERSION = "\(.*\)";$/\1/p' scripts/fetch-onnxruntime.mjs)" + [ -n "$VERSION" ] || { echo "::error::VERSION not found in scripts/fetch-onnxruntime.mjs"; exit 1; } + COMMIT="$(sed -n 's/^const SOURCE_COMMIT = "\(.*\)";$/\1/p' scripts/fetch-onnxruntime.mjs)" + [ -n "$COMMIT" ] || { echo "::error::SOURCE_COMMIT not found in scripts/fetch-onnxruntime.mjs"; exit 1; } + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "commit=$COMMIT" >> "$GITHUB_OUTPUT" + echo "Building ONNX Runtime v$VERSION at $COMMIT" + + - name: Read the deployment floor from electron-builder.json5 + id: floor + # Also single-sourced: if somebody raises `mac.minimumSystemVersion`, this + # build follows rather than silently producing a library for the old floor. + run: | + set -euo pipefail + FLOOR="$(grep -o '"minimumSystemVersion": *"[0-9.]*"' electron-builder.json5 | grep -o '[0-9][0-9.]*')" + [ -n "$FLOOR" ] || { echo "::error::minimumSystemVersion not found"; exit 1; } + echo "floor=$FLOOR" >> "$GITHUB_OUTPUT" + echo "Deployment target: $FLOOR" + + - name: Checkout ONNX Runtime + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: microsoft/onnxruntime + # The COMMIT, not the tag. `v1.27.1` upstream is a lightweight tag — it points + # straight at a commit and can be moved. Building from a tag would mean + # attesting an artifact to "whatever that tag meant this morning", which is + # precisely the property this workflow exists to provide. + ref: ${{ steps.pin.outputs.commit }} + path: onnxruntime-src + submodules: recursive + fetch-depth: 1 + persist-credentials: false + + - name: Check the pinned commit is still what the tag names + # Not fatal to the build — the commit above is what gets built either way — but a + # tag that has moved means the pin and the version string no longer describe the + # same thing, and somebody should look before adopting the artifact. + run: | + set -euo pipefail + TAGGED="$(git -C onnxruntime-src ls-remote https://github.com/microsoft/onnxruntime "refs/tags/v${{ steps.pin.outputs.version }}" | cut -f1)" + if [ "$TAGGED" != "${{ steps.pin.outputs.commit }}" ]; then + echo "::error::v${{ steps.pin.outputs.version }} now resolves to ${TAGGED:-nothing}, not the pinned ${{ steps.pin.outputs.commit }}" + exit 1 + fi + echo "v${{ steps.pin.outputs.version }} still resolves to ${{ steps.pin.outputs.commit }}" + + - name: Read the runner image + id: image + # `${{ env.ImageOS }}` DOES NOT WORK, and it fails silently. The `env` + # expression context only carries what a workflow, job or step `env:` block + # defined; `ImageOS`/`ImageVersion` are set by the runner in its own + # environment, so the expression evaluates to the empty string and the cache + # key simply loses that component. The evidence is in this repo's own cache + # list: `build-whisper-stt.yml` builds its key the same way and the stored + # keys read `whisper-stt-build-darwin-arm64---` — three hyphens, both + # values empty. Reading them in a `run:` step, where they are ordinary shell + # variables, is what actually works. + run: | + set -euo pipefail + echo "tag=${ImageOS:-unknown}-${ImageVersion:-unknown}" >> "$GITHUB_OUTPUT" + echo "Runner image: ${ImageOS:-unknown} ${ImageVersion:-unknown}" + + - name: Cache the CMake build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: onnxruntime-build + # Keyed on the version, the floor and the runner image. The image matters: + # CMake bakes absolute SDK paths into the tree, so when GitHub rolls Xcode a + # restored tree fails on paths that no longer exist. Scoping the key to the + # image busts it automatically on every roll. + key: ort-${{ steps.pin.outputs.version }}-${{ steps.floor.outputs.floor }}-${{ steps.image.outputs.tag }} + + - name: Configure + # Driven straight at CMake rather than through upstream's `build.sh`. + # `build.sh` -> `build_args.py` uses `match`, so it needs Python 3.10+, + # and `cmake/CMakeLists.txt` asks for `find_package(Python 3.10)` — but + # that requirement is only real for the Python bindings, which are not + # built here. The one thing Python IS needed for is generating the symbol + # export list (`onnxruntime.lds`); pointing `Python_EXECUTABLE` at + # whatever the runner has is enough, and `gen_def.py` parses on 3.9. + run: | + set -euo pipefail + cmake -S onnxruntime-src/cmake -B onnxruntime-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ steps.floor.outputs.floor }} \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -Donnxruntime_BUILD_SHARED_LIB=ON \ + -Donnxruntime_BUILD_UNIT_TESTS=OFF \ + -DPython_EXECUTABLE="$(command -v python3)" + + - name: Build + # BOUNDED, deliberately. Bare `--parallel` means "as many jobs as cores", and + # ONNX Runtime's C++ translation units are memory-hungry: on a runner with far + # less RAM per core than the reference M1, that is how a 10-minute build becomes + # an hour of swapping. `nproc`-1 leaves the machine a core to breathe. + run: | + set -euo pipefail + JOBS="$(( $(sysctl -n hw.ncpu) > 2 ? $(sysctl -n hw.ncpu) - 1 : 1 ))" + echo "Building with $JOBS jobs on $(sysctl -n hw.ncpu) cores, $(( $(sysctl -n hw.memsize) / 1073741824 )) GiB" + cmake --build onnxruntime-build --config Release --parallel "$JOBS" + + - name: Verify the deployment target + # The entire reason this workflow exists. A library that comes out at 14.0 + # anyway is worse than no library, because it would sail through packaging + # and strand macOS 13 users at dyld time. + run: | + set -euo pipefail + V="${{ steps.pin.outputs.version }}" + DYLIB="onnxruntime-build/libonnxruntime.${V}.dylib" + [ -f "$DYLIB" ] || { echo "::error::$DYLIB was not produced"; exit 1; } + MINOS="$(otool -l "$DYLIB" | awk '/LC_BUILD_VERSION/{f=1} f&&/minos/{print $2; exit}')" + echo "minos=$MINOS floor=${{ steps.floor.outputs.floor }}" + [ "$MINOS" = "${{ steps.floor.outputs.floor }}" ] || { + echo "::error::built for macOS $MINOS, expected ${{ steps.floor.outputs.floor }}"; exit 1; } + + - name: Verify the ABI surface + # `ort` is wired `load-dynamic`, so it dlopens this file and calls + # `OrtGetApiBase`. The CPU provider is the one `segmentation.rs` uses. + # CoreML is deliberately absent and is NOT checked for. + run: | + set -euo pipefail + V="${{ steps.pin.outputs.version }}" + DYLIB="onnxruntime-build/libonnxruntime.${V}.dylib" + for sym in _OrtGetApiBase _OrtSessionOptionsAppendExecutionProvider_CPU; do + nm -gU "$DYLIB" | grep -q " ${sym}$" || { echo "::error::missing export ${sym}"; exit 1; } + done + lipo -info "$DYLIB" + echo "ABI surface OK" + + - name: Package in the upstream layout + # Byte-for-byte the same shape as `onnxruntime-osx-arm64-.tgz`, so + # `fetch-onnxruntime.mjs` needs no extraction change — only a URL and a + # digest. `member` there is the VERSIONED file; the two symlinks beside it + # are kept so the archive stays a drop-in for anything that expects them. + run: | + set -euo pipefail + V="${{ steps.pin.outputs.version }}" + DIR="onnxruntime-osx-arm64-${V}" + mkdir -p "stage/${DIR}/lib" "stage/${DIR}/include" + cp "onnxruntime-build/libonnxruntime.${V}.dylib" "stage/${DIR}/lib/" + ln -s "libonnxruntime.${V}.dylib" "stage/${DIR}/lib/libonnxruntime.dylib" + ln -s "libonnxruntime.${V}.dylib" "stage/${DIR}/lib/libonnxruntime.1.dylib" + cp onnxruntime-src/include/onnxruntime/core/session/*.h "stage/${DIR}/include/" || true + # NOT optional. `fetch-onnxruntime.mjs` refuses an archive with no LICENSE + # (`LICENSE not found inside …`) and then reads it to confirm the library + # really is MIT — "asset names are not evidence". Without this the archive + # would pass its SHA-256 and fail at vendoring, which is the worst place to + # find out. `ThirdPartyNotices.txt` rides along because upstream ships it and + # THIRD-PARTY-NOTICES.md is what carries the attribution. + cp onnxruntime-src/LICENSE "stage/${DIR}/LICENSE" + cp onnxruntime-src/ThirdPartyNotices.txt "stage/${DIR}/ThirdPartyNotices.txt" + tar -czf "${DIR}.tgz" -C stage "${DIR}" + shasum -a 256 "${DIR}.tgz" + + - name: Attest build provenance + id: attest + # A SHA-256 in `fetch-onnxruntime.mjs` says "these are the bytes somebody + # pinned". It cannot say WHERE they came from — and once the publisher is us + # rather than Microsoft, that is the question that matters. This binds the + # archive's digest to the commit, workflow and run that produced it, signed by + # GitHub, so provenance becomes verifiable rather than asserted: + # + # gh attestation verify onnxruntime-osx-arm64-.tgz --repo getopenscreen/openscreen + # + # It does not replace the digest pin, it answers a different question. Keep both. + uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2.4.0 + with: + subject-path: onnxruntime-osx-arm64-*.tgz + + - name: Upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: onnxruntime-osx-arm64 + path: onnxruntime-osx-arm64-*.tgz + if-no-files-found: error + retention-days: 90 + + - name: Workflow summary + if: always() + shell: bash + run: | + set -euo pipefail + V="${{ steps.pin.outputs.version }}" + ARCHIVE="onnxruntime-osx-arm64-${V}.tgz" + { + echo "## ONNX Runtime ${V}, macOS arm64, deployment target ${{ steps.floor.outputs.floor }}" + echo "" + echo "- Source: microsoft/onnxruntime@\`${{ steps.pin.outputs.commit }}\`" + echo "" + echo "- Result: ${{ job.status }}" + } >> "$GITHUB_STEP_SUMMARY" + if [ "${{ steps.attest.outcome }}" != "success" ]; then + { + echo "" + echo "> **Provenance was NOT attested** (\`${{ steps.attest.outcome }}\`)." + echo "> Do not adopt this archive: the digest below says what the bytes are," + echo "> nothing says where they came from. Re-run the workflow." + } >> "$GITHUB_STEP_SUMMARY" + fi + if [ -f "$ARCHIVE" ] && [ "${{ steps.attest.outcome }}" = "success" ]; then + SHA="$(shasum -a 256 "$ARCHIVE" | cut -d' ' -f1)" + { + echo "- Archive: \`${ARCHIVE}\` ($(du -h "$ARCHIVE" | cut -f1))" + echo "- SHA-256: \`${SHA}\`" + echo "" + echo "Provenance is attested; before adopting, verify it with:" + echo "" + echo '```bash' + echo "gh attestation verify ${ARCHIVE} --repo ${{ github.repository }}" + echo '```' + echo "" + echo "To adopt it: attach the archive to a release, then point the" + echo "\`darwin-arm64\` entry of \`PINNED\` in \`scripts/fetch-onnxruntime.mjs\`" + echo "at that release with this digest:" + echo "" + echo '```js' + echo '"darwin-arm64": {' + echo ' slug: "osx-arm64",' + echo ' ext: "tgz",' + echo " sha256: \"${SHA}\"," + echo " member: \`libonnxruntime.\${VERSION}.dylib\`," + echo ' out: "libonnxruntime.dylib",' + echo ' baseUrl: "",' + echo '},' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/build-whisper-stt.yml b/.github/workflows/build-whisper-stt.yml index 6f5b3de5b..9fe27584f 100644 --- a/.github/workflows/build-whisper-stt.yml +++ b/.github/workflows/build-whisper-stt.yml @@ -16,6 +16,13 @@ name: Build whisper-stt binaries on: workflow_dispatch: push: + # ALL branches, NO tags. `paths:` alone also matches a tag push, and it has: + # publishing `v0.0.0-onnxruntime-1.27.1` started a four-platform whisper build + # for a tag that touched none of these files. Listing `branches` is what + # excludes tags; `'**'` keeps every branch working, which is the point of this + # workflow — contributors push a branch to get binaries built. + branches: + - "**" paths: - "scripts/build-whisper-stt.sh" - "electron/native/whisper-stt/**" @@ -139,6 +146,23 @@ jobs: "${VCPKG_ROOT_DIR}/vcpkg" install spirv-headers:x64-windows echo "CMAKE_PREFIX_PATH=${VCPKG_ROOT_DIR}/installed/x64-windows" >> "$GITHUB_ENV" + # `${{ env.ImageOS }}` DOES NOT WORK, and it fails silently — the `env` + # expression context holds only what a workflow/job/step `env:` block put + # there, and the runner sets ImageOS/ImageVersion into its own process + # environment instead. Written that way, both halves expanded to the empty + # string and the cache below was keyed on the tag and the CMakeLists hash + # alone, for every platform, for as long as the line existed: the repo's + # own cache list read `whisper-stt-build-darwin-arm64---`. Reading + # them here in a shell is what actually gets their values. + # + # `shell: bash` is required, not decorative: this matrix includes + # windows-latest, where the default shell is PowerShell and `${VAR:-default}` + # is not syntax. GitHub ships bash on the Windows image. + - name: Read the runner image + id: image + shell: bash + run: echo "tag=${ImageOS:-unknown}-${ImageVersion:-unknown}" >> "$GITHUB_OUTPUT" + - name: Cache whisper.cpp build tree uses: actions/cache@v6 with: @@ -151,15 +175,15 @@ jobs: # bump there invalidates the cache instead of silently reusing a stale # FetchContent checkout; falls back to the newest cache for the same # platform + runner image on a miss so incremental compilation still - # helps. The runner image version ($ImageOS/$ImageVersion) is part of + # helps. The runner image version (read by the step above) is part of # the key AND the restore-keys prefix because CMake bakes absolute # toolchain paths (e.g. the Xcode SDK's libz.tbd) into the cached build # tree — when GitHub rolls the image's Xcode/SDK, those paths vanish and # a restored tree fails with "No rule to make target …libz.tbd". Scoping # the cache to the image version auto-busts it on every toolchain roll. - key: whisper-stt-build-${{ matrix.tag }}-${{ env.ImageOS }}-${{ env.ImageVersion }}-${{ hashFiles('electron/native/whisper-stt/CMakeLists.txt') }} + key: whisper-stt-build-${{ matrix.tag }}-${{ steps.image.outputs.tag }}-${{ hashFiles('electron/native/whisper-stt/CMakeLists.txt') }} restore-keys: | - whisper-stt-build-${{ matrix.tag }}-${{ env.ImageOS }}-${{ env.ImageVersion }}- + whisper-stt-build-${{ matrix.tag }}-${{ steps.image.outputs.tag }}- - name: Run whisper-stt build script run: bash scripts/build-whisper-stt.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d6ecbfb6..9b3beb623 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,19 @@ on: push: tags: - "v*" + # `v0.0.0-*` is this repository's marker for a tag that is NOT a product + # version — a place to hang a binary that needs a permanent public URL, the + # way `v0.0.0-stt-models` hosts the 360 MB Whisper model. Without this + # exclusion, creating one runs the whole matrix — Windows, macOS x2, Linux, + # installers, notarisation — to publish an archive nobody asked it to build. + # That has already happened once. + # + # The other five release-triggered workflows do not need the same guard: + # AUR, Nix, winget and Homebrew all gate on + # `!github.event.release.prerelease`, and an internal release is marked as a + # prerelease. Only the Discord announcement still fires, which is noise + # rather than a wrong publication. + - "!v0.0.0-*" workflow_dispatch: inputs: arch: @@ -353,6 +366,16 @@ jobs: - name: Build Metal compositor addon run: npm run build:native:compositor:mac + # Third step this job has to spell out, same reason as the two above: it is in + # `npm run build:mac`, which this job does not run — it needs `--dir` plus a + # hand-rolled DMG. Windows and Linux get it free from `build:win` / `build:linux`. + # Unlike the compositor addon, a missing ONNX Runtime does not fail the pack: the + # camera-background control just does nothing, on every shipped Mac, with nothing + # in CI raising a word. No-ops on x64 — upstream publishes no osx-x64 asset, so the + # script says so and exits 0 rather than failing that build. + - name: Stage ONNX Runtime + run: npm run fetch:onnxruntime + - name: Package .app bundle run: npx electron-builder --mac --${{ matrix.arch }} --dir --publish never env: @@ -988,6 +1011,12 @@ jobs: # with a tag ref and failed its deploy every time. See docs.yml. if: ${{ steps.release.outputs.is_prerelease == 'false' }} timeout-minutes: 20 + # Bookkeeping, not publishing: the release itself is already out by the time + # this runs. Letting it fail the job silently skipped publish-msstore on + # v1.10.0 (an unrelated Store submission, gated on this job's `needs` success) + # even though "Publish release assets" above had already succeeded. Allowed + # to fail so a flaky docs deploy never blocks a downstream publish step. + continue-on-error: true env: GH_TOKEN: ${{ secrets.OPENSCREEN_RELEASE_TOKEN }} run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fcb0c20b..5ee421be9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,9 +159,19 @@ jobs: # [env] block (cargo has no [target..env] — the macOS section in that file # is inert and cargo warns "unused key"), so before that change this job pointed # bindgen at the win64 tree and could never have gone green. + # + # Without a library on ORT_DYLIB_PATH, `runtime_available()` is false and every + # segmentation test returns early — the suite goes green having exercised no + # inference at all, which is exactly how the `ort`-panics-when-absent bug got + # in. Staging it here is what makes `the_whole_loop_produces_a_mask_from_ + # compose_frame_alone` a real test on this runner instead of a skipped one. + # ~30 MB, next to nothing beside `brew install ffmpeg` above. + - name: Stage ONNX Runtime + run: node scripts/fetch-onnxruntime.mjs - name: cargo test (compositor, aarch64-apple-darwin) env: MAC_FFMPEG_DIR: /opt/homebrew/opt/ffmpeg + ORT_DYLIB_PATH: ${{ github.workspace }}/electron/native/bin/darwin-arm64/libonnxruntime.dylib run: | cd crates cargo test -p openscreen-compositor --lib --tests @@ -266,11 +276,21 @@ jobs: # et rendre une erreur qui ne designe pas la cause non plus. test -n "$libclang" || { echo "libclang introuvable apres l'installation"; exit 1; } echo "LIBCLANG_PATH=$(dirname "$libclang")" >> "$GITHUB_ENV" + # Meme raison que sur le job macOS : sans bibliotheque sur ORT_DYLIB_PATH, + # `runtime_available()` est faux et chaque test de segmentation rend la main + # tout de suite — la suite passe au vert sans avoir exerce la moindre + # inference, ce qui est exactement par ou le bug « ort panique quand elle + # manque » est entre. C'est ce qui fait de + # `the_whole_loop_produces_a_mask_from_compose_frame_alone` un vrai test ici + # plutot qu'un test saute. Builtins node uniquement, comme fetch:ffmpeg:sdk. + - name: Stage ONNX Runtime + run: node scripts/fetch-onnxruntime.mjs - name: cargo test (compositor) env: # Les .so ffmpeg vendorises ne sont dans aucun chemin systeme : sans ca # le binaire de test se lance puis meurt sur `libavformat.so.62`. LD_LIBRARY_PATH: ${{ github.workspace }}/crates/thirdparty/ffmpeg-linux64-lgpl-shared/lib + ORT_DYLIB_PATH: ${{ github.workspace }}/electron/native/bin/linux-x64/libonnxruntime.so # Fait ECHOUER `cpu_backend_linux.rs` s'il n'obtient pas le backend CPU, # au lieu de le sauter en silence comme sur un poste sans lavapipe. OPENSCREEN_REQUIRE_CPU_BACKEND: "1" diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index a8531c9ac..e2ae200cb 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -24,6 +24,25 @@ name: Nix build # front of every merge. Promote it once the schedule has reported a few times. on: workflow_dispatch: + # The job that actually builds the derivation now runs on the pull requests that + # can break it. It did not, and the gap was not academic: `nix-check.yml` only + # compares npmDepsHash, so a PR rewriting the addon's source filter, its RPATH + # handling or its symbols.map went green on ~18 checks without one of them + # building it, and the first real signal arrived on main half an hour after the + # merge. #371 shipped a change to `nix/compositor-view.nix` that way. + # + # Path-filtered rather than universal: this takes about half an hour, and a PR + # that touches none of these files cannot change what it produces. + pull_request: + paths: + - flake.nix + - flake.lock + - nix/** + - crates/** + - package-lock.json + # Including itself, or a PR that only edits this file gets no validation of + # the change it is making -- the same rule nix-check.yml already follows. + - .github/workflows/nix-build.yml push: branches: [main] schedule: @@ -56,7 +75,11 @@ concurrency: # That is the affordable half. Verifying each merge would need a queue this # workflow does not have, and is not worth it for a half-hour job whose purpose # is catching drift rather than gating a commit. - cancel-in-progress: false + # + # On a pull request the opposite is right: a new push makes the previous run + # answer a question nobody is asking any more, and at half an hour each they + # would pile up. `github.ref` is the PR's merge ref, so the group is per-PR. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: build: @@ -421,84 +444,100 @@ jobs: fi # The real acceptance test. Everything above proves the package starts - # and can list a screen; none of it touches the compositor addon, which - # is what actually renders output. Record a couple of seconds, export it, - # and look at what came out. + # and can answer an enumeration call; none of it touches the compositor + # addon, which is what actually renders output. # - # This block was briefly moved ahead of the sources loop and moved back, - # so that it is not tried a third time. The theory was that position - # explained why record seemed to fail far more often than sources -- - # run_cli spawns a fresh `xvfb-run -a` each time, so record was always - # invocations 6-8, after five Xvfb servers had come and gone. The - # experiment could not answer it: by the time it ran, record had started - # succeeding from its old position anyway, so there was no contrast left - # to measure. From position 4 it succeeded, which proves nothing it was - # not already doing from position 9. + # It used to record two seconds and export the result. That never once + # worked here. `record` needs a display index and this host has no + # display to give: Chromium's X11 capturer logs + # "screen_capturer_x11.cc: Failed to initialize pixel buffer", `sources` + # answers with `displays: []`, and record dies on "Display index 0 not + # found (0 screen(s) available)". So export never ran, and the step that + # exists to vouch for the addon vouched for nothing -- runs 32707512544 + # (24/08) and 32827253816 (25/08) failed exactly here, in the 34-36 + # minutes they took to get past the build. # - # What the runs did establish is that the premise was wrong. Enumeration - # here is bimodal -- 12-31ms when it answers, no return at all when it - # does not, with nothing in between across every measurement so far -- - # and the failures cluster by run and by window within a run rather than - # by command. The apparent record-versus-sources gap was that clustering - # seen through a denominator, not a property of either path. Reopen this - # with the run_cli labels, on a run that actually fails, before assuming - # otherwise. + # Measured before rewriting it, against the built artefact under Xvfb: + # with `xvfb-run -a` as this workflow invokes it, 4 runs in 10 saw a + # display; starting Xvfb by hand and polling xdpyinfo until the server + # answered before launching the app, 3 in 10. So it is not the startup + # race it looks like -- waiting for the server changes nothing -- and + # whatever it is lives inside Chromium's X11 capturer. On this runner it + # comes up zero every time rather than a third of the time. # - # Up to three goes, because screen capture on this host is unreliable in - # its own right. One success is enough for the question being asked here. - echo "--- record then export (first run_cli here is #$((RUN_CLI_N + 1))) ---" - EXPORTED="" - # Tracked apart from EXPORTED so the verdict can name the stage that - # actually failed. For three runs every attempt died in record without - # export ever executing, while the annotation said "the export path does - # not work" -- an accusation aimed at the one component the run never - # reached, and the compositor addon is precisely what this step exists - # to vouch for. - RECORDED=0 - for i in 1 2 3; do - echo "=== export attempt $i/3 (run_cli #$((RUN_CLI_N + 1))) ===" - rm -f /tmp/demo.openscreen /tmp/demo.mp4 - RC=0 - CLI_TIMEOUT=120 OPENSCREEN_DIAGNOSTIC=1 run_cli $SANDBOX $CHROME_FLAGS record --duration 2 --project /tmp/demo.openscreen >"/tmp/rec.$i.out" 2>&1 || RC=$? - # Outside the failure branch for the same reason as above: a record that - # works is exactly the measurement missing from the comparison, since - # this path has never yet produced one. - grep -a "get-sources\]" "/tmp/rec.$i.out" || true - if [ "$RC" -ne 0 ] || [ ! -f /tmp/demo.openscreen ]; then - echo "record failed (rc=$RC); last lines:" - tail -5 "/tmp/rec.$i.out" || true - continue - fi - RECORDED=1 - echo "recorded. project:" - head -c 200 /tmp/demo.openscreen; echo + # So the input stops being a recording. ffmpeg synthesises two seconds of + # video, a three-line project points at it, and export renders that. + # Identical in what it proves -- the packaged compositor addon loads, + # decodes, composes through Vulkan and muxes an MP4 -- and it asks for no + # capability a headless runner is ever going to have. Measured at 6/6 + # locally where record measured 4/10. + # + # Capture is still worth watching, so one probe still runs. It is + # informational: it cannot pass here, and nothing gates on it. + echo "--- capture probe (informational; run_cli #$((RUN_CLI_N + 1))) ---" + RC=0 + CLI_TIMEOUT=120 OPENSCREEN_DIAGNOSTIC=1 run_cli $SANDBOX $CHROME_FLAGS record --duration 2 --project /tmp/probe.openscreen >/tmp/probe.out 2>&1 || RC=$? + if [ "$RC" -eq 0 ] && [ -f /tmp/probe.openscreen ]; then + echo "::warning::record worked on this runner. Capture is no longer broken here -- see whether the export check below should go back to using a real recording." + else + echo "capture still unavailable (rc=$RC); last lines:" + tail -3 /tmp/probe.out || true + fi - RC=0 - CLI_TIMEOUT=180 OPENSCREEN_DIAGNOSTIC=1 run_cli $SANDBOX $CHROME_FLAGS export /tmp/demo.openscreen -o /tmp/demo.mp4 >"/tmp/exp.$i.out" 2>&1 || RC=$? - if [ "$RC" -ne 0 ] || [ ! -f /tmp/demo.mp4 ]; then - echo "export failed (rc=$RC); last lines:" - tail -15 "/tmp/exp.$i.out" || true - continue - fi - EXPORTED=/tmp/demo.mp4 - break - done + echo "--- synthesise a clip and export it (first run_cli here is #$((RUN_CLI_N + 1))) ---" + # From the flake's own nixpkgs, for the same reason the Vulkan ICD is: + # the ambient registry drifts, and a decoder that is never the same twice + # is drift injected into a check that exists to catch it. + FFMPEG=$(nix build --no-link --print-out-paths --inputs-from . nixpkgs#ffmpeg-headless) + FFMPEG=${FFMPEG%%$'\n'*} + echo "ffmpeg: $FFMPEG" + + # H.264 in MP4 with an AAC track: the shape a real recording arrives in, + # so the export walks its ordinary decode path rather than a special one. + "$FFMPEG/bin/ffmpeg" -loglevel error -y \ + -f lavfi -i "testsrc2=size=1280x720:rate=30" \ + -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -t 2 -pix_fmt yuv420p -c:v libx264 -c:a aac -shortest /tmp/demo-src.mp4 + ls -l /tmp/demo-src.mp4 + # The whole project format the exporter needs: a media path and an empty + # editor, which normalises to a single full-length clip. This is what + # `record --project` writes, minus the parts a recording fills in. + cat > /tmp/demo.openscreen <<'JSON' + { + "version": 2, + "media": { "screenVideoPath": "/tmp/demo-src.mp4" }, + "editor": {} + } + JSON + # Parse it back before handing it over, so a future edit that breaks the + # JSON fails here with a parse error rather than 300 s later as an + # export that could not read its project. + python3 -c "import json;json.load(open('/tmp/demo.openscreen'))" + cat /tmp/demo.openscreen + + echo "--- openscreen info ---" + CLI_TIMEOUT=120 run_cli $SANDBOX $CHROME_FLAGS info /tmp/demo.openscreen || true + + rm -f /tmp/demo.mp4 + RC=0 + CLI_TIMEOUT=300 OPENSCREEN_DIAGNOSTIC=1 run_cli $SANDBOX $CHROME_FLAGS export /tmp/demo.openscreen -o /tmp/demo.mp4 >/tmp/exp.out 2>&1 || RC=$? EXPORT_OK=0 - if [ -z "$EXPORTED" ] && [ "$RECORDED" -eq 0 ]; then - echo "::error::No attempt got past record, so export never ran and the compositor addon is unproven. This is a capture failure on this host, not an export failure." - elif [ -z "$EXPORTED" ]; then - echo "::error::record produced a project but no attempt produced an MP4. The compositor addon is packaged and the export path does not work." + if [ "$RC" -ne 0 ] || [ ! -f /tmp/demo.mp4 ]; then + echo "::error::export failed (rc=$RC). The compositor addon is packaged and the export path does not work." + tail -25 /tmp/exp.out || true else - SIZE=$(wc -c < "$EXPORTED") + SIZE=$(wc -c < /tmp/demo.mp4) # An MP4 opens with a 4-byte length then 'ftyp'. A zero-length or # truncated file would otherwise pass a mere existence check. - MAGIC=$(dd if="$EXPORTED" bs=1 skip=4 count=4 2>/dev/null || true) + MAGIC=$(dd if=/tmp/demo.mp4 bs=1 skip=4 count=4 2>/dev/null || true) echo "exported $SIZE bytes, magic at offset 4: $MAGIC" if [ "$MAGIC" != "ftyp" ]; then echo "::error::output is not an MP4 (no ftyp box)" + tail -25 /tmp/exp.out || true elif [ "$SIZE" -lt 10000 ]; then echo "::error::MP4 is only $SIZE bytes, too small to hold two seconds of video" + tail -25 /tmp/exp.out || true else echo "Export works: $SIZE bytes of MP4." EXPORT_OK=1 @@ -509,7 +548,7 @@ jobs: # flaky must not hide whether export works, which is the whole point of # having packaged the compositor addon. # - # The gate is "did enumeration ever work" and "does export work", not + # The gate is "did enumeration ever answer" and "does export work", not # "did all five attempts pass". Requiring FAILED -eq 0 made the job red # by construction: the standing numbers on this runner are 1/5, 3/5 and # 4/5 ok, so a run where export is perfect and four enumerations succeed @@ -518,7 +557,11 @@ jobs: # per-attempt warnings above keep that flakiness visible without letting # it decide the build; tighten this to $ATTEMPTS once the capture failure # is understood and fixed. - echo "=== verdict: enumeration $OK/$ATTEMPTS ok, record $RECORDED, export $EXPORT_OK, $RUN_CLI_N run_cli invocations ===" + # + # Export, on the other hand, is a hard gate on every trigger again. It no + # longer depends on a capability this host does not have, so there is + # nothing left to excuse: if it fails now, the package is broken. + echo "=== verdict: enumeration $OK/$ATTEMPTS ok, export $EXPORT_OK, $RUN_CLI_N run_cli invocations ===" if [ "$EXPORT_OK" -ne 1 ] || [ "$OK" -eq 0 ]; then exit 1 fi diff --git a/.gitignore b/.gitignore index a1822ee05..24ad425eb 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ workbench/fixtures/ /aur_ci /aur_ci.pub /aur_known_hosts +tmp_handoff.md diff --git a/README.md b/README.md index 88c05f3de..7f197284a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ License Latest Release CI Status - Discord + Discord + Sponsor Platform

@@ -62,6 +63,15 @@ See [docs/cli.md](./docs/cli.md). Every platform has a recommended route below. On Windows that is the Microsoft Store; everywhere else it is the installer from the [GitHub Releases](https://github.com/getopenscreen/openscreen/releases) page. +### System requirements + +- **Windows**: version 1903+ (build 18362) with Intel 8th Gen / AMD Ryzen 2000 series or newer minimum; Windows 11 with Intel 12th Gen / Ryzen 4000 series or newer recommended +- **macOS**: 13 (Ventura) or later — required by ScreenCaptureKit for capture +- **Linux**: `xdg-desktop-portal` and PipeWire for native capture and system audio; recording still works without them through the browser-capture fallback, with fewer capabilities (see [Platform differences](#platform-differences)) +- **RAM**: 8 GB minimum, 16 GB recommended + +Full table and notes on older integrated graphics: [system requirements](https://getopenscreen.com/docs/installation#system-requirements). + ### macOS Download the `.dmg` installer directly from the [Releases page](https://github.com/getopenscreen/openscreen/releases) and drag OpenScreen into your Applications folder. Builds from 1.9.0 onward are signed with a Developer ID certificate and notarized by Apple, so Gatekeeper does not block them and no terminal step is needed. @@ -162,10 +172,10 @@ You may need to grant screen recording permissions depending on your desktop env Everything in the editor and export is the same on macOS, Windows, and Linux: zooms, backgrounds, motion blur, crop/trim/speed, blur regions, annotations, auto-captions, AI editing, projects, export, and all languages. All three now record through a native capture pipeline; the remaining differences are narrower than they used to be: - **Native recording**: macOS (ScreenCaptureKit), Windows (Windows Graphics Capture), and Linux (PipeWire via the ScreenCast portal) all record through a native pipeline for higher quality and clean window-level capture. On Linux the browser pipeline stays as an automatic fallback if the helper isn't available. -- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. Linux captures position and cursor shape through the portal, so cursor themes and the editable cursor overlay work there too — but the portal reports no mouse button events, so **click effects remain macOS and Windows only**. +- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. Linux captures position and cursor shape through the portal, so cursor themes and the editable cursor overlay work there too. Click effects work on Linux as well, but not through the portal — Wayland exposes no portal for mouse buttons, so the capture helper reads the left button from evdev, which needs your user in the `input` group. Without that, recording is unaffected and every cursor sample is simply a move. - **Webcam**: Windows muxes the webcam natively into the recording; macOS and Linux record it alongside as a separate file. It works as a picture-in-picture overlay on all three. - **System audio** support varies by OS: - - **macOS**: requires macOS 13+. On macOS 14.2+ you'll be prompted to grant audio capture permission. macOS 12 and below can't capture system audio (mic still works). + - **macOS**: works on every supported version. On macOS 14.2+ you'll be prompted to grant audio capture permission. - **Windows**: works out of the box. - **Linux**: needs PipeWire (default on Ubuntu 22.04+, Fedora 34+). Older PulseAudio-only setups may not capture system audio (mic should still work). @@ -185,9 +195,10 @@ For safety, download OpenScreen only from the official GitHub Releases linked fr OpenScreen is community-driven. If you need help, want to report a bug, or just want to chat with other users and contributors: -- 💬 **Discord** — [Join the OpenScreen Discord](https://discord.gg/VvT6Vtnyh) for real-time help, showcase, and discussion +- 💬 **Discord** — [Join the OpenScreen Discord](https://getopenscreen.com/discord) for real-time help, showcase, and discussion - 🐞 **[GitHub Issues](https://github.com/getopenscreen/openscreen/issues)** — bug reports and feature requests - 🗺️ **[Roadmap](./ROADMAP.md)** — see what we're building next +- ❤️ **Support the project** — [GitHub Sponsors](https://github.com/sponsors/EtienneLescot) or [Ko-fi](https://ko-fi.com/etiennelescot) --- diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index 330bf7492..6bfe88fb4 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -49,19 +49,62 @@ distributed by their own registries, not redistributed inside our binaries. - The speech model (`ggml-*.bin`) is **not** bundled — it is downloaded into the user's data directory on first use by `electron/stt/modelManager.ts`. -## Microsoft OpenMP runtime — `vcomp140.dll` (Windows only) +## ONNX Runtime (Windows and Apple Silicon macOS) -- **Component**: `resources/electron/native/bin/win32-x64/vcomp140.dll`. +- **Component**: `onnxruntime.dll` / `libonnxruntime.dylib`, under + `resources/electron/native/bin/-/`. +- **License**: MIT — . +- Not built here: the pinned upstream release archive is downloaded, SHA-256 + verified and unpacked by `scripts/fetch-onnxruntime.mjs`, which also checks the + archive's own LICENSE really is MIT before vendoring anything. +- **Why it ships**: the native compositor segments the webcam subject with it, on + the CPU execution provider, to drive the camera background cutout/blur/custom + modes. The `gpu_cuda*` builds are deliberately not used — they are an order of + magnitude larger and carry NVIDIA redistribution terms. +- **Not on Intel macOS**: upstream publishes no `osx-x86_64` asset from 1.27 on, + so the x64 DMG ships without it and the camera background effects are simply + absent there. Not shipped on Linux either, where the compositor has no capture + path for the mask yet. +- The segmentation model it runs is a separate component, immediately below. + +## MediaPipe Selfie Segmentation — model weights + +- **Components**: `selfie_segmentation.tflite`, + `selfie_segmentation_landscape.tflite` and the `selfie_segmentation_landscape.onnx` + derived from them, shipped inside `app.asar` under `dist/mediapipe/`. +- **License**: Apache-2.0 — . + Copyright The MediaPipe Authors. +- The `.onnx` is a **derived work**, generated from the vendored `.tflite` by + `scripts/convert-selfie-segmentation-to-onnx.py`. No third-party weights are + downloaded at build time. +- **Why it is listed here**: these weights are redistributed inside the installer, + and Apache-2.0 §4 asks that the attribution travel with them. The provenance note + in `public/mediapipe/selfie_segmentation/README.md` does not — electron-builder's + `"!*.md"` filter strips it from the package — so this file is the only copy a user + ever receives. +- The MediaPipe **JavaScript** solution and its two ~5.6 MB WASM builds are no longer + bundled: inference moved into the native compositor, and nothing loaded them. + +## Microsoft Visual C++ runtime — `vcomp140.dll`, `msvcp140*.dll`, `vcruntime140*.dll` (Windows only) + +- **Components**: under `resources/electron/native/bin/win32-x64/` — + `vcomp140.dll`, `msvcp140.dll`, `msvcp140_1.dll`, `vcruntime140.dll`, + `vcruntime140_1.dll`. - **License**: redistributable under the Microsoft Visual C++ Redistributable - terms accompanying Visual Studio; the copy shipped is taken from the - `VC\Redist\MSVC\\x64\Microsoft.VC.OpenMP\` directory of the - Visual Studio installation that builds the release, never from `System32`. -- **Why it ships**: the ggml backends above are compiled with OpenMP and import - it. It is **not** part of Windows, so without it `whisper-stt-server` dies in - the loader before `main()` on any machine that has no Visual C++ - Redistributable, and transcription and captions fail with no usable error. - Staged by `scripts/stage-vcomp-runtime.mjs`; `scripts/before-pack.cjs` refuses - to package if it is missing while anything still imports it. + terms accompanying Visual Studio; the copies shipped are taken from the + `VC\Redist\MSVC\\x64\Microsoft.VC.OpenMP\` and + `…\Microsoft.VC.CRT\` directories of the Visual Studio installation that + builds the release, never from `System32`. +- **Why they ship**: two prebuilt binaries in the payload import them, and + neither is ours to recompile against the static CRT. The ggml backends above + are compiled with OpenMP and import `vcomp140.dll`; the vendored ONNX Runtime + imports the CRT proper. None of these are **part of Windows**, so without them + `whisper-stt-server` dies in the loader before `main()` on any machine that has + no Visual C++ Redistributable — transcription and captions fail with no usable + error — and `onnxruntime.dll` fails to load, leaving the camera background + silently inert. Staged by `scripts/stage-vcomp-runtime.mjs`; + `scripts/before-pack.cjs` refuses to package if any is missing while something + still imports it. ## PipeWire — headers (Linux only) diff --git a/biome.json b/biome.json index 4fa1d2fdc..8954e61bf 100644 --- a/biome.json +++ b/biome.json @@ -3,7 +3,7 @@ "vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true }, "files": { "ignoreUnknown": false, - "includes": ["**", "!**/*.css", "!**/design/**", "!**/.worktrees/**"] + "includes": ["**", "!**/*.css", "!**/design/**", "!**/.worktrees/**", "!**/public/mediapipe/**"] }, "formatter": { "enabled": true, diff --git a/crates/.cargo/config.toml b/crates/.cargo/config.toml index 29197773e..cb1fd42fb 100644 --- a/crates/.cargo/config.toml +++ b/crates/.cargo/config.toml @@ -1,5 +1,10 @@ # FFMPEG_DIR relatif au dossier crates/ (portable dans le repo). Y déposer le build # ffmpeg LGPL-shared (voir README). LIBCLANG_PATH = install LLVM locale (bindgen). +# Ces deux valeurs sont celles de WINDOWS et cargo n'a pas de `[target..env]` : +# le `[env]` ci-dessous est global, donc elles sont posées sur les trois OS. C'est +# `crates/compositor/build.rs` qui les neutralise ailleurs — voir +# `point_libclang_at_the_xcode_toolchain()` (macOS) et `drop_unusable_libclang_path()` +# (Linux). Ne pas supposer qu'un `LIBCLANG_PATH` non vide désigne un vrai libclang. # Ces valeurs cèdent à une vraie variable d'environnement (force=false par défaut). # # Pinné sur le MÊME build release-branch (n8.1.2-34-g9b6c8969e0, tag BtbN diff --git a/crates/Cargo.lock b/crates/Cargo.lock index dbe2f253a..da41d7070 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -44,7 +44,7 @@ version = "0.38.0+1.3.281" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" dependencies = [ - "libloading", + "libloading 0.8.9", ] [[package]] @@ -180,7 +180,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.8.9", ] [[package]] @@ -652,7 +652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" dependencies = [ "libc", - "libloading", + "libloading 0.8.9", "pkg-config", ] @@ -678,6 +678,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -720,6 +730,16 @@ dependencies = [ "libc", ] +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.8.3" @@ -867,7 +887,37 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3" dependencies = [ - "libloading", + "libloading 0.8.9", +] + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", ] [[package]] @@ -889,6 +939,24 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -918,6 +986,7 @@ name = "openscreen-compositor" version = "0.0.0" dependencies = [ "anyhow", + "ash", "bindgen", "block", "cc", @@ -925,11 +994,14 @@ dependencies = [ "cosmic-text", "image", "metal 0.29.0", + "ndarray 0.16.1", "objc", + "ort", "pollster", "serde", "serde_json", "wgpu", + "wgpu-hal", "windows", ] @@ -942,6 +1014,25 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ort" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" +dependencies = [ + "libloading 0.9.0", + "ndarray 0.17.2", + "ort-sys", + "smallvec", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1011,6 +1102,21 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "presser" version = "0.3.1" @@ -1075,6 +1181,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "read-fonts" version = "0.37.0" @@ -1434,6 +1546,25 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "ttf-parser" version = "0.25.1" @@ -1630,7 +1761,7 @@ dependencies = [ "js-sys", "khronos-egl", "libc", - "libloading", + "libloading 0.8.9", "log", "metal 0.31.0", "naga", diff --git a/crates/Cargo.toml b/crates/Cargo.toml index 208f376cc..abd74df99 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -34,6 +34,26 @@ image = { version = "0.25", default-features = false, features = ["jpeg", "png"] wgpu = { version = "24", features = ["wgsl"] } pollster = "0.4" cosmic-text = "0.19" +# Inference for the webcam segmentation mask. CPU execution provider only: measured on the +# target integrated GPU it costs +0.47 ms/frame against DirectML's +1.03, its cost does not +# scale with input resolution, and choosing it deletes the whole D3D11<->D3D12 interop +# (technical-documentation/engineering/webcam-segmentation.md). +# +# Behind the `segmentation` feature and OFF by default: `download-binaries` fetches the +# ONNX Runtime libs at build time, which is a packaging decision (nix, AUR, MS Store, CI) +# that has not been taken yet. The default build is unchanged. +# `load-dynamic` et NON `download-binaries` : ce dernier tire une build STATIQUE d'ONNX +# Runtime avec DirectML dedans (DirectML.lib, DXCORE.lib et les DmlOperator* apparaissent +# dans la ligne de lien) — exactement la dépendance que le choix de l'EP CPU sert à +# supprimer. En chargement dynamique, la lib est résolue à l'exécution, ce qui laisse le +# packaging la stager par plateforme comme il le fait déjà pour whisper-stt. +ort = { version = "2.0.0-rc.13", default-features = false, features = [ + "std", + "ndarray", + "load-dynamic", + "api-27", +] } +ndarray = "0.16" [workspace.dependencies.windows] version = "0.58" diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 62c164e0c..69476ffc2 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -70,6 +70,22 @@ pub fn probe_backend() -> String { .to_string() } +/// Si cette machine peut produire un masque de segmentation, c'est-à-dire si la bibliothèque +/// ONNX Runtime est là où l'app l'a posée. +/// +/// Sert à NE PAS MENTIR : le contrôle « fond de caméra » est le seul de l'éditeur dont l'effet +/// dépend d'un binaire optionnel. Sans lui, `Segmenter::load` refuse, le compositeur dessine la +/// webcam telle quelle, et l'utilisateur clique sur un réglage qui ne fait rien — exactement ce +/// qu'un contrôle ne doit jamais faire. +/// +/// Une question posée au système plutôt que devinée depuis la plateforme : `darwin` ne suffit +/// pas à répondre, puisque l'amont ne publie aucun binaire ONNX pour les Macs Intel, et une +/// build de dev ou un `--dir` n'en ont pas davantage. Seul l'état réel de la machine le sait. +#[napi] +pub fn segmentation_runtime_available() -> bool { + openscreen_compositor::segmentation::runtime_available() +} + #[napi] pub fn create_view( rect: CompositorViewRect, diff --git a/crates/compositor/Cargo.toml b/crates/compositor/Cargo.toml index f968b5497..face980ee 100644 --- a/crates/compositor/Cargo.toml +++ b/crates/compositor/Cargo.toml @@ -12,12 +12,40 @@ path = "src/lib.rs" bindgen = "0.70" cc = "1" +[features] +default = ["segmentation"] +# Segmentation IA de la webcam via ONNX Runtime (EP CPU). +# +# Activée par défaut, ce qui ne coûte rien au build : `ort` est lié en `load-dynamic`, donc +# aucune bibliothèque n'est nécessaire pour COMPILER. Elle l'est pour tourner — absente, +# `Segmenter::load` échoue, le compositeur écrit une ligne et dessine la webcam telle quelle. +# La désactiver reste possible pour une build qui ne veut pas du tout du code d'inférence. +segmentation = ["dep:ort", "dep:ndarray"] + [dependencies] anyhow.workspace = true +wgpu.workspace = true +pollster.workspace = true +cosmic-text.workspace = true +ort = { workspace = true, optional = true } +ndarray = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true image.workspace = true +# Acces Vulkan brut, UNIQUEMENT pour ouvrir le device avec les extensions de +# memoire externe (cf. `d3d_linux::open_device_with_dmabuf_export`). Les versions +# sont celles que wgpu 24 tire deja : en prendre d'autres ferait cohabiter deux +# bindings pour un meme `VkDevice`. +# +# LINUX SEULEMENT, et rien d'autre ne doit venir ici : ces deux lignes ont ete +# ajoutees sous cet en-tete par une edition qui l'a place trop haut, emportant +# `wgpu`, `serde`, `ort` et le reste avec elles. Windows et macOS ont alors cesse +# de compiler. +[target.'cfg(target_os = "linux")'.dependencies] +ash = "0.38" +wgpu-hal = { version = "24", features = ["vulkan"] } + # Windows : D3D11 + D3D11VA + Direct2D/DirectWrite + HLSL à l'exécution. [target.'cfg(windows)'.dependencies] windows.workspace = true @@ -47,7 +75,3 @@ core-foundation = "0.9" # Linux : wgpu (Vulkan) pour le rendu, pollster pour block_on les ops wgpu, # cosmic-text pour la rastérisation du texte (remplace DirectWrite/CoreText). -[target.'cfg(target_os = "linux")'.dependencies] -wgpu.workspace = true -pollster.workspace = true -cosmic-text.workspace = true diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index f8c9e5f5b..99b11e2f6 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -10,6 +10,8 @@ fn main() { if target_is_macos { point_libclang_at_the_xcode_toolchain(); + } else if target_os == "linux" { + drop_unusable_libclang_path(); } // Le pin ffmpeg est porté par `.cargo/config.toml` ; sur Windows c'est le @@ -29,28 +31,25 @@ fn main() { // un dev l'a posé à la main pour macOS, jamais quand il vient du pin Windows. env::var("MAC_FFMPEG_DIR") .ok() - .filter(|v| Path::new(v).join("include").exists()) - .or_else(|| { - // `thirdparty/` est frère de `compositor/`, sous `crates/` — c'est aussi - // ce que le pin Windows désigne (`relative = true` dans - // `crates/.cargo/config.toml`, relatif au dossier de la config). - // build.rs s'exécute avec cwd = racine du crate, pas `crates/`, donc on - // remonte depuis CARGO_MANIFEST_DIR plutôt que d'écrire un chemin relatif - // qui viserait `crates/compositor/thirdparty/`. - let candidate = Path::new(&env::var("CARGO_MANIFEST_DIR").ok()?) - .parent()? - .join("thirdparty") - .join("ffmpeg-n8.1.2-macos64-lgpl-shared"); - candidate - .join("include") - .exists() - .then(|| candidate.to_string_lossy().to_string()) - }) + .filter(|v| usable_ffmpeg_tree(Path::new(v))) + .or_else(|| vendored_ffmpeg_tree("ffmpeg-n8.1.2-macos64-lgpl-shared")) .or_else(|| { env::var("FFMPEG_DIR") .ok() - .filter(|v| Path::new(v).join("include").exists()) + .filter(|v| usable_ffmpeg_tree(Path::new(v))) }) + } else if target_os == "linux" { + // Même piège que LIBCLANG_PATH, même remède : le `FFMPEG_DIR` du `[env]` global + // désigne l'arbre win64, qui n'existe pas ici, donc on ne l'accepte que s'il + // pointe sur un arbre RÉEL — c'est-à-dire quand un dev ou + // scripts/build-linux-compositor-addon.mjs l'a posé à la main. Sinon on retombe + // sur l'emplacement vendorisé conventionnel, dans le même ordre que ce script + // (`resolveFfmpegDir`), pour qu'un `cargo check` nu et un build via npm voient + // le même arbre. + env::var("FFMPEG_DIR") + .ok() + .filter(|v| usable_ffmpeg_tree(Path::new(v))) + .or_else(|| vendored_ffmpeg_tree("ffmpeg-linux64-lgpl-shared")) } else { env::var("FFMPEG_DIR").ok() }; @@ -58,9 +57,12 @@ fn main() { let include_dir = match ff.as_ref() { Some(v) => Path::new(v).join("include").to_string_lossy().to_string(), None => panic!( - "crates/compositor build.rs: FFMPEG_DIR non défini (target={}). \ + "crates/compositor build.rs: aucun arbre ffmpeg utilisable (target={}). \ Sur Windows, voir crates/.cargo/config.toml. Sur macOS, poser \ - MAC_FFMPEG_DIR ou vendoriser thirdparty/ffmpeg-n8.1.2-macos64-lgpl-shared.", + MAC_FFMPEG_DIR ou vendoriser thirdparty/ffmpeg-n8.1.2-macos64-lgpl-shared. \ + Sur Linux, poser FFMPEG_DIR ou vendoriser \ + thirdparty/ffmpeg-linux64-lgpl-shared (arbre *shared*, avec include/ et \ + lib/ — celui de scripts/fetch-ffmpeg.mjs est statique et ne convient pas).", target_os ), }; @@ -69,7 +71,7 @@ fn main() { if let Some(v) = ff.as_ref() { let lib_dir = Path::new(v).join("lib"); println!("cargo:rustc-link-search=native={}", lib_dir.display()); - for lib in ["avformat", "avcodec", "avutil", "swscale", "swresample"] { + for lib in ["avformat", "avcodec", "avutil", "swscale", "swresample", "avfilter"] { println!("cargo:rustc-link-lib=dylib={}", lib); } } @@ -316,3 +318,109 @@ fn point_libclang_at_the_xcode_toolchain() { None => env::remove_var("LIBCLANG_PATH"), } } + +/// Un arbre ffmpeg exploitable : `include/` pour bindgen ET `lib/` pour le linkage. +/// +/// Les deux, pas seulement le premier : la section « linkage » plus bas pose un +/// `rustc-link-search` sur `/lib` et réclame avformat/avcodec/avutil/swscale/ +/// swresample. Un arbre n'ayant que les en-têtes passait le filtre, écartait le repli +/// vers un arbre vendorisé complet, et échouait bien plus tard sur un `cannot find +/// -lavformat` qui ne désigne pas sa cause. `resolveFfmpegDir()` dans +/// scripts/build-linux-compositor-addon.mjs vérifie déjà les deux — c'est la même règle +/// des deux côtés. +fn usable_ffmpeg_tree(dir: &Path) -> bool { + dir.join("include").is_dir() && dir.join("lib").is_dir() +} + +/// L'arbre ffmpeg vendorisé sous `crates/thirdparty/`, s'il existe vraiment. +/// +/// `thirdparty/` est frère de `compositor/`, sous `crates/` — c'est aussi ce que le pin +/// Windows désigne (`relative = true` dans `crates/.cargo/config.toml`, relatif au +/// dossier de la config). build.rs s'exécute avec cwd = racine du crate, pas `crates/`, +/// donc on remonte depuis CARGO_MANIFEST_DIR plutôt que d'écrire un chemin relatif qui +/// viserait `crates/compositor/thirdparty/`. +fn vendored_ffmpeg_tree(name: &str) -> Option { + let candidate = Path::new(&env::var("CARGO_MANIFEST_DIR").ok()?) + .parent()? + .join("thirdparty") + .join(name); + usable_ffmpeg_tree(&candidate).then(|| candidate.to_string_lossy().to_string()) +} + +/// Même remède que le versant macOS, pour la même cause. +/// +/// `crates/.cargo/config.toml` pose `LIBCLANG_PATH` dans un `[env]` GLOBAL, faute de +/// `[target..env]` en cargo. La valeur est celle de Windows +/// (`C:\Program Files\LLVM\bin`) et elle est donc renseignée sous Linux aussi, où +/// clang-sys la prend au mot : il ne regarde nulle part ailleurs et abandonne sur +/// « Unable to find libclang », alors qu'un `libclang.so` de distribution est presque +/// toujours installé. Un `cargo check -p openscreen-compositor` nu échouait donc sur +/// une Ubuntu de série — exactement ce que `freestanding_header_args()` juste au-dessus +/// s'emploie à éviter par ailleurs. +/// +/// On ne devine pas le bon chemin : clang-sys sait chercher tout seul (LD_LIBRARY_PATH, +/// PATH, /usr/lib/llvm-*/lib …). Il suffit de ne pas lui mentir. Une valeur posée par le +/// dev et réellement utilisable est conservée telle quelle — `force = false` fait déjà +/// gagner l'environnement réel sur la config, et on ne casse pas un choix explicite. +fn drop_unusable_libclang_path() { + println!("cargo:rerun-if-env-changed=LIBCLANG_PATH"); + let Ok(value) = env::var("LIBCLANG_PATH") else { + return; + }; + // clang-sys accepte DEUX formes : un fichier bibliothèque, ou un répertoire qui en + // contient un (`search_libclang_directories` : « Check if the path is a matching + // file », puis « … a directory containing a matching file »). Ne traiter que le + // répertoire retirerait un `LIBCLANG_PATH` parfaitement valide pointant sur + // `/usr/lib/llvm-N/lib/libclang.so.1`. + let path = Path::new(&value); + let usable = if path.is_file() { + path.file_name() + .is_some_and(|n| is_libclang_filename(&n.to_string_lossy())) + } else { + std::fs::read_dir(path).is_ok_and(|entries| { + entries + .flatten() + // `is_file()` autant que le nom : `read_dir` rend aussi les + // sous-répertoires et les fichiers spéciaux, et un répertoire qui + // s'appellerait `libclang.so` passerait le seul test de nom — clang-sys + // le retiendrait puis échouerait à le charger, sans repli possible. + .any(|e| { + e.path().is_file() && is_libclang_filename(&e.file_name().to_string_lossy()) + }) + }) + }; + if !usable { + env::remove_var("LIBCLANG_PATH"); + } +} + +/// Les motifs EXACTS que clang-sys cherche sous Linux : `libclang.so`, +/// `libclang-.so`, `libclang.so.`, `libclang-.so.`. +/// +/// Coller aux motifs, et pas seulement au préfixe, parce que `search_libclang_directories` +/// s'arrête net sur `LIBCLANG_PATH` quand la variable est posée — « Search only the path +/// indicated by the relevant environment variable » — sans jamais retomber sur +/// `llvm-config`, le PATH ou les répertoires connus. Conserver un chemin qui ne contient +/// qu'un `libclang_extra.so` ou un `libclang.software` reviendrait donc à condamner le +/// build, exactement comme le faisait la valeur Windows. +/// +/// `libclang-cpp.*` est écarté d'entrée : clang-sys l'écarte lui-même +/// (`filename.contains("-cpp.")`), `libclang_shared` ayant été renommé `libclang-cpp` à +/// partir de Clang 10. +fn is_libclang_filename(name: &str) -> bool { + if name.contains("-cpp.") { + return false; + } + let Some(rest) = name.strip_prefix("libclang") else { + return false; + }; + // Soit `libclang.so…`, soit `libclang-.so…` avec un `` non vide. + let rest = match rest.strip_prefix('-') { + Some(versioned) => match versioned.find(".so") { + None | Some(0) => return false, + Some(i) => &versioned[i..], + }, + None => rest, + }; + rest == ".so" || rest.starts_with(".so.") +} diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index db9b46e13..ee90c1374 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -3,8 +3,9 @@ //! unique encodeur AAC alimente le même muxer que la vidéo. use crate::ffi::*; + use crate::regions::SpeedSegment; -use crate::scene::SceneAudio; +use crate::scene::{SceneAudio, SceneAudioTrack}; use anyhow::{bail, Result}; use std::f32::consts::PI; use std::ffi::CString; @@ -477,6 +478,12 @@ pub struct WsolaTimeStretcher { buf: PlanarPcm, mono: Vec, buf_start: i64, + /// Décalage de lecture dans `buf`/`mono`. `discard_below` ne recopiait pas moins que le + /// reste du buffer à chaque grain : la région entière est poussée d'un coup, donc pour + /// 65 M d'échantillons cela faisait ~N²/(2·ha) ≈ 1,1e12 f32 recopiés par canal — le vrai + /// coût du chemin WSOLA, devant la recherche par grain. On avance un curseur et on ne + /// compacte que lorsque la tête dépasse la moitié du buffer, ce qui rend le total O(N). + buf_head: usize, out: PlanarPcm, win_sum: Vec, out_start: usize, @@ -522,6 +529,7 @@ impl WsolaTimeStretcher { buf: vec![Vec::new(); channels], mono: Vec::new(), buf_start: 0, + buf_head: 0, out: vec![Vec::new(); channels], win_sum: Vec::new(), out_start: 0, @@ -587,8 +595,12 @@ impl WsolaTimeStretcher { } } + fn buf_len(&self) -> usize { + self.buf[0].len() - self.buf_head + } + fn buf_end(&self) -> i64 { - self.buf_start + self.buf[0].len() as i64 + self.buf_start + self.buf_len() as i64 } fn sample_at(&self, channel: usize, absolute_index: i64) -> f32 { @@ -596,7 +608,10 @@ impl WsolaTimeStretcher { if index < 0 { 0.0 } else { - self.buf[channel].get(index as usize).copied().unwrap_or(0.0) + self.buf[channel] + .get(self.buf_head + index as usize) + .copied() + .unwrap_or(0.0) } } @@ -605,12 +620,19 @@ impl WsolaTimeStretcher { if index < 0 { 0.0 } else { - self.mono.get(index as usize).copied().unwrap_or(0.0) + self.mono + .get(self.buf_head + index as usize) + .copied() + .unwrap_or(0.0) } } fn process(&mut self, final_chunk: bool) -> PlanarPcm { let mut emitted = self.empty_chunk(); + // Pas de garde anti-stagnation ici : `search_target` croît de `ha > 0` à chaque tour + // et `grain_pos` ne s'en écarte que de `search_radius` au plus, donc le break sur + // `buf_end` finit toujours par tomber. Une garde de plus tronquerait `emitted` — la + // région sortirait muette pour tout signal — sans jamais se déclencher. loop { let search_target = (self.ideal_pos + self.ha).round() as i64; let required_end = (self.grain_pos + self.n as i64) @@ -738,12 +760,18 @@ impl WsolaTimeStretcher { if drop_count <= 0 { return; } - let drop_count = drop_count as usize; - for channel in 0..self.channels { - self.buf[channel] = self.buf[channel][drop_count.min(self.buf[channel].len())..].to_vec(); - } - self.mono = self.mono[drop_count.min(self.mono.len())..].to_vec(); + self.buf_head += (drop_count as usize).min(self.buf_len()); self.buf_start = absolute_index; + // Compactage amorti : ne recopier que lorsque la tête consommée dépasse ce qui + // reste laisse un coût total en O(N) au lieu du O(N²) d'une recopie par grain. + if self.buf_head > self.buf[0].len() - self.buf_head { + let head = self.buf_head; + for channel in 0..self.channels { + self.buf[channel].drain(..head); + } + self.mono.drain(..head); + self.buf_head = 0; + } } } @@ -767,6 +795,20 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { } let speed = source_samples as f64 / target_samples as f64; + + // atempo d'abord : le WSOLA ci-dessous fait le même time-stretch préservant la hauteur, + // mais coûte un ordre de grandeur de plus. Mesuré en release sur une région de 5 min + // (14,4 M échantillons) : 0,6 s contre 20 s à 1,25×, 4,9 s contre 55 s à 0,25× — et + // c'est le WSOLA APRÈS la correction de `discard_below`, qui recopiait tout le buffer + // restant à chaque grain et faisait tenir un export mesuré (65,4 M échantillons) plus de + // dix minutes sans finir, l'export paraissant figé à ~80 %. `avfilter_atempo_stretch` + // rend `None` si la chaîne ne monte pas, si le sink négocie un format inattendu ou si la + // sortie reste plus courte que la cible ; le WSOLA reste alors le chemin de repli exact + // d'avant, en journalisant la raison. + if let Some(stretched) = unsafe { avfilter_atempo_stretch(pcm, target_samples, speed) } { + return stretched; + } + let mut stretcher = WsolaTimeStretcher::new( AUDIO_OUTPUT_SAMPLE_RATE, AUDIO_OUTPUT_CHANNELS, @@ -792,6 +834,431 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { exact } +/// Plafond du nombre d'étages atempo chaînés. +/// +/// `speed` vient de `source_samples / target_samples`, pas de l'éditeur : une scène corrompue +/// où une poignée d'échantillons vise une cible d'une heure donne un ratio arbitrairement +/// petit, et le chaînage par 0.5 empile alors une trentaine d'étages — plus d'un millier pour +/// un subnormal — chacun avec sa fenêtre d'analyse et sa perte d'amorçage. Huit couvre +/// jusqu'à 0.5⁸ ≈ 0,0039, soit vingt-cinq fois sous `MIN_PLAYBACK_SPEED` (0,1) ; au-delà on +/// rend `None` et le WSOLA, qui n'a pas de bornes, prend le relais. +const ATEMPO_MAX_STAGES: usize = 8; + +/// Découpe un facteur de vitesse en facteurs que `atempo` accepte individuellement : le +/// filtre n'admet que [0.5, 100.0], on chaîne donc les dépassements (0.2 → [0.5, 0.5, 0.8], +/// 250 → [100.0, 2.5]) — le produit des facteurs reconstitue la vitesse demandée. +/// +/// Rend `None` au-delà de `ATEMPO_MAX_STAGES` maillons. La borne haute est chaînée elle +/// aussi : `MAX_PLAYBACK_SPEED` vaut 100 donc un seul étage suffit à tout ce que l'éditeur +/// produit, mais `speed` est un rapport de longueurs quantifiées, pas la vitesse cliquée, et +/// rien ne garantit qu'il reste sous la borne du filtre. +fn atempo_factors(speed: f64) -> Option> { + let mut factors = Vec::new(); + let mut remaining = speed; + while remaining > 100.0 || remaining < 0.5 { + if factors.len() >= ATEMPO_MAX_STAGES { + return None; + } + if remaining > 100.0 { + factors.push(100.0); + remaining /= 100.0; + } else { + factors.push(0.5); + remaining /= 0.5; + } + } + factors.push(remaining); + Some(factors) +} + +/// RAII : libère le graphe même en sortie précoce sur erreur. +struct FilterGraphGuard(*mut AVFilterGraph); + +impl Drop for FilterGraphGuard { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { avfilter_graph_free(&mut self.0) }; + } + } +} + +/// RAII : libère la trame de drain même en sortie précoce sur erreur. +struct FrameGuard(*mut AVFrame); + +impl Drop for FrameGuard { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { av_frame_free(&mut self.0) }; + } + } +} + +/// Taille des trames poussées vers le graphe. Le drain est entrelacé avec l'alimentation +/// (cf. `atempo_drain`) : sans cela `av_buffersrc_add_frame` empile toute la région dans la +/// file du buffersrc — ~523 Mo pour une speed region stéréo de 20 min, en plus du slice +/// d'entrée et de l'accumulateur de sortie. +const ATEMPO_FEED_CHUNK: usize = 4096; + +/// Rallonge de silence poussée derrière la région avant l'EOF, par étage atempo. +/// +/// atempo laisse tomber la dernière fenêtre de chaque étage. En prolongeant l'entrée d'un +/// silence, la fenêtre perdue devient du silence et le contenu réel sort en entier : mesuré +/// sur le pin ffmpeg n8.1.2 (48 kHz stéréo), le manque tombe de 981 à 217 échantillons pour +/// un étage 0.5×, de 2 735 à 553 pour deux, de 8 234 à 2 676 pour quatre. Au-delà la courbe +/// est plate — un tail 16× plus grand ne change plus rien — ce qui reste est traité par la +/// correction de tempo de `avfilter_atempo_stretch`. +const ATEMPO_PRIME_TAIL: usize = 4096; + +/// Marge de sécurité, en échantillons, sur la longueur demandée à la passe corrigée. +/// +/// Le manque de la seconde passe n'est pas exactement celui mesuré à la première (le tempo +/// a bougé de moins de 1 %, la chaîne est la même). Viser 64 échantillons de plus fait +/// tomber le résidu du côté du surplus, tronqué : 1,3 ms de contenu en moins plutôt qu'un +/// trou de silence. +const ATEMPO_LENGTH_GUARD: usize = 64; + +/// Longueur du silence à pousser derrière la région pour une chaîne donnée. +fn atempo_prime_tail(factors: &[f64], speed: f64) -> usize { + ATEMPO_PRIME_TAIL + .saturating_mul(factors.len() + 1) + .saturating_mul(speed.max(1.0).ceil() as usize) +} + +/// Vide le buffersink dans `stretched`, sans jamais y garder plus de `keep` échantillons par +/// plan, et compte dans `produced` TOUT ce qui est sorti — y compris ce qui est jeté. +/// +/// Les deux chiffres servent à des choses différentes : `stretched` est le résultat, alors +/// que `produced` mesure ce que la chaîne a réellement rendu pour une entrée de longueur +/// connue, donc son manque (cf. `avfilter_atempo_stretch`). +/// +/// Rend `Some(true)` sur EOF, `Some(false)` quand le graphe n'a plus rien de prêt (EAGAIN), +/// `None` sur une vraie panne — l'appelant retombe alors sur WSOLA. +unsafe fn atempo_drain( + sink_ctx: *mut AVFilterContext, + frame: *mut AVFrame, + stretched: &mut PlanarPcm, + keep: usize, + produced: &mut usize, +) -> Option { + loop { + let ret = av_buffersink_get_frame(sink_ctx, frame); + if ret == AVERROR_EAGAIN { + return Some(false); + } + if ret == AVERROR_EOF { + return Some(true); + } + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: av_buffersink_get_frame a échoué (ret={ret}), repli WSOLA" + ); + return None; + } + let count = (*frame).nb_samples.max(0) as usize; + let channels = (*frame).ch_layout.nb_channels.max(0) as usize; + // La chaîne est épinglée en flt entrelacé de bout en bout (cf. `avfilter_atempo_stretch`) ; + // tout autre format signifie que la négociation a fait autre chose que ce qu'on a + // demandé, et le désentrelacement ci-dessous lirait n'importe quoi. + if (*frame).format != AVSampleFormat::AV_SAMPLE_FMT_FLT as i32 + || channels != AUDIO_OUTPUT_CHANNELS + { + eprintln!( + "[openscreen-compositor] atempo: trame de sortie inattendue (format={} canaux={channels}), repli WSOLA", + (*frame).format + ); + av_frame_unref(frame); + return None; + } + let wanted = count.min(keep.saturating_sub(stretched[0].len())); + if wanted > 0 { + let interleaved = *(*frame).extended_data.add(0) as *const f32; + let samples = + std::slice::from_raw_parts(interleaved, count * AUDIO_OUTPUT_CHANNELS); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let plane = &mut stretched[channel]; + plane.reserve(wanted); + for index in 0..wanted { + plane.push(samples[index * AUDIO_OUTPUT_CHANNELS + channel]); + } + } + } + *produced += count; + av_frame_unref(frame); + } +} + +/// Monte `abuffer → atempo… → abuffersink`, y pousse `pcm` suivi de `prime_tail` échantillons +/// de silence, et rend le nombre total d'échantillons sortis — `stretched` en reçoit les +/// `keep` premiers. +/// +/// La chaîne est épinglée en **flt entrelacé** 48 kHz stéréo, pas en fltp : `af_atempo` +/// n'annonce que des formats packed (U8/S16/S32/FLT/DBL, cf. son `query_formats`), donc un +/// abuffer en fltp fait insérer un aresample de conversion et rend une branche planaire du +/// drain inatteignable — mesuré, le sink négociait déjà `AV_SAMPLE_FMT_FLT`. En demandant flt +/// des deux côtés il n'y a aucun filtre de conversion dans le graphe, et l'entrelacement est +/// absorbé par la recopie qu'on fait de toute façon. +unsafe fn atempo_pass( + pcm: &[Vec], + factors: &[f64], + prime_tail: usize, + keep: usize, + stretched: &mut PlanarPcm, +) -> Option { + let graph_guard = FilterGraphGuard(avfilter_graph_alloc()); + let graph = graph_guard.0; + if graph.is_null() { + return None; + } + + let abuffer_name = CString::new("abuffer").ok()?; + let abuffersink_name = CString::new("abuffersink").ok()?; + let atempo_name = CString::new("atempo").ok()?; + let abuffer = avfilter_get_by_name(abuffer_name.as_ptr()); + let abuffersink = avfilter_get_by_name(abuffersink_name.as_ptr()); + let atempo = avfilter_get_by_name(atempo_name.as_ptr()); + if abuffer.is_null() || abuffersink.is_null() || atempo.is_null() { + return None; + } + + let create_filter = |graph: *mut AVFilterGraph, + filter: *const AVFilter, + name: &str, + args: Option<&str>, + options: &[(&str, &str)]| + -> Option<*mut AVFilterContext> { + let cname = CString::new(name).ok()?; + let cargs = match args { + Some(args) => Some(CString::new(args).ok()?), + None => None, + }; + let ctx = avfilter_graph_alloc_filter(graph, filter, cname.as_ptr()); + if ctx.is_null() { + eprintln!("[openscreen-compositor] atempo: alloc_filter({name}) a rendu null"); + return None; + } + // Les options typées se posent entre l'alloc et l'init — `avfilter_init_str` fige la + // négociation. Un échec n'est pas fatal : l'abuffer porte déjà le format, ceci ne fait + // que l'imposer aussi côté sink pour qu'aucun build ffmpeg ne puisse y glisser un + // aresample. Le drain vérifie le format reçu de toute façon. + for (key, value) in options { + let ckey = CString::new(*key).ok()?; + let cvalue = CString::new(*value).ok()?; + let ret = av_opt_set( + ctx as *mut std::ffi::c_void, + ckey.as_ptr(), + cvalue.as_ptr(), + AV_OPT_SEARCH_CHILDREN as i32, + ); + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: av_opt_set({name}.{key}={value}) a échoué (ret={ret}), négociation laissée libre" + ); + } + } + // `map_or` consommerait `cargs` et le pointeur rendu par la closure serait dangling + // avant même l'appel — on emprunte donc pour la durée de l'appel. + let args_ptr = match &cargs { + Some(args) => args.as_ptr(), + None => ptr::null(), + }; + let ret = avfilter_init_str(ctx, args_ptr); + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: init_str({name}, {:?}) a échoué (ret={ret})", + args.unwrap_or("") + ); + return None; + } + Some(ctx) + }; + + let rate = AUDIO_OUTPUT_SAMPLE_RATE; + let src_ctx = create_filter( + graph, + abuffer, + "in", + Some(&format!( + "time_base=1/{rate}:sample_rate={rate}:sample_fmt=flt:channel_layout=stereo" + )), + &[], + )?; + let sink_ctx = create_filter(graph, abuffersink, "out", None, &[("sample_fmts", "flt")])?; + + let mut previous = src_ctx; + for (index, factor) in factors.iter().enumerate() { + let stage = create_filter( + graph, + atempo, + &format!("atempo{index}"), + Some(&format!("{factor}")), + &[], + )?; + if avfilter_link(previous, 0, stage, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_link a échoué au maillon {index}"); + return None; + } + previous = stage; + } + if avfilter_link(previous, 0, sink_ctx, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_link vers le sink a échoué"); + return None; + } + if avfilter_graph_config(graph, ptr::null_mut()) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_graph_config a échoué"); + return None; + } + + let sink_frame = FrameGuard(av_frame_alloc()); + if sink_frame.0.is_null() { + return None; + } + + // Alimentation : le PCM passe par trames flt de 4096 échantillons, prolongé par la + // rallonge de silence. `av_buffersrc_add_frame` déplace les références du frame dans le + // graphe ; on alloue donc une trame neuve par tranche et on la libère après envoi (le + // shell est vide à ce point). Le drain est entrelacé ici : sans lui la file du buffersrc + // porterait toute la région d'un coup. La condition d'arrêt est l'entrée épuisée, PAS + // « on a de quoi remplir la cible » — s'arrêter là couperait la rallonge, et les derniers + // grains du contenu réel resteraient dans le graphe. + let source_samples = pcm.first().map(|plane| plane.len()).unwrap_or(0); + let total_input = source_samples.saturating_add(prime_tail); + let mut offset = 0usize; + let mut produced = 0usize; + let mut drained_to_eof = false; + while offset < total_input { + let count = ATEMPO_FEED_CHUNK.min(total_input - offset); + let mut frame = av_frame_alloc(); + if frame.is_null() { + eprintln!("[openscreen-compositor] atempo: av_frame_alloc (feed) a échoué"); + return None; + } + (*frame).format = AVSampleFormat::AV_SAMPLE_FMT_FLT as i32; + (*frame).sample_rate = rate; + (*frame).nb_samples = count as i32; + av_channel_layout_default(&mut (*frame).ch_layout, AUDIO_OUTPUT_CHANNELS as i32); + if av_frame_get_buffer(frame, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: av_frame_get_buffer (feed) a échoué"); + av_frame_free(&mut frame); + return None; + } + // Un seul plan en flt : on écrit entrelacé. Le `write_bytes` couvre à la fois les + // canaux absents d'une source mono et la rallonge de silence finale. + let destination = *(*frame).extended_data.add(0) as *mut f32; + ptr::write_bytes(destination, 0, count * AUDIO_OUTPUT_CHANNELS); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + if let Some(plane) = pcm.get(channel) { + let available = plane.len().saturating_sub(offset).min(count); + for index in 0..available { + *destination.add(index * AUDIO_OUTPUT_CHANNELS + channel) = + plane[offset + index]; + } + } + } + (*frame).pts = offset as i64; + let ret = av_buffersrc_add_frame(src_ctx, frame); + av_frame_free(&mut frame); + if ret < 0 { + eprintln!("[openscreen-compositor] atempo: av_buffersrc_add_frame (offset={offset}) a échoué (ret={ret})"); + return None; + } + offset += count; + if atempo_drain(sink_ctx, sink_frame.0, stretched, keep, &mut produced)? { + drained_to_eof = true; + break; + } + } + + // EOF : le graphe vide alors ses derniers grains. + if !drained_to_eof { + if av_buffersrc_add_frame(src_ctx, ptr::null_mut()) < 0 { + eprintln!("[openscreen-compositor] atempo: flush du buffersrc a échoué, repli WSOLA"); + return None; + } + atempo_drain(sink_ctx, sink_frame.0, stretched, keep, &mut produced)?; + } + Some(produced) +} + +/// Étire le PCM d'un facteur `speed` via une chaîne `abuffer → atempo… → abuffersink` montée +/// en processus, dans l'avfilter LGPL déjà vendored avec l'app (avfilter-11.dll / +/// libavfilter.so.11 / libavfilter.11.dylib voyagent dans le même lot que avcodec — cf. +/// scripts/fetch-ffmpeg.mjs qui copie TOUTES les av*.dll du build BtbN). +/// +/// **Deux passes.** atempo ne rend pas exactement `n/tempo` échantillons : il en manque un +/// nombre fixe par chaîne, indépendant de la longueur de l'entrée (mesuré sur n8.1.2 : +/// ~217 pour un étage, ~550 pour deux, ~2 700 pour quatre, soit jusqu'à 56 ms à 0,1×). Le +/// manque ne se rattrape pas en poussant plus d'entrée — c'est une différence de durée +/// rendue, pas une queue retenue — et le combler par des zéros collait un trou de silence +/// devant le segment suivant, puisque le crossfade equal-power ne couvre que les frontières +/// de clip, jamais la concaténation par segment. La première passe mesure donc le manque sur +/// le contenu réel, sans rien garder, et la seconde demande `cible + manque` pour que le +/// contenu remplisse la cible ; le surplus est tronqué. Aux vitesses > 1 le manque est nul et +/// la seconde passe est sautée. +/// +/// Retourne `None` sur toute défaillance (montage, négociation, exécution, sortie plus courte +/// que la cible) : l'appelant retombe alors sur le WSOLA d'origine. +unsafe fn avfilter_atempo_stretch( + pcm: &[Vec], + target_samples: usize, + speed: f64, +) -> Option { + if !speed.is_finite() || speed <= 0.0 || target_samples == 0 { + return None; + } + let source_samples = pcm.first().map(|plane| plane.len()).unwrap_or(0); + if source_samples == 0 { + return None; + } + + let planes = |capacity: usize| -> PlanarPcm { + (0..AUDIO_OUTPUT_CHANNELS) + .map(|_| Vec::with_capacity(capacity)) + .collect() + }; + + let factors = atempo_factors(speed)?; + let prime_tail = atempo_prime_tail(&factors, speed); + let mut stretched = planes(target_samples); + let produced = atempo_pass(pcm, &factors, prime_tail, target_samples, &mut stretched)?; + let expected = ((source_samples + prime_tail) as f64 / speed).round() as usize; + let shortfall = expected.saturating_sub(produced); + + if shortfall > 0 { + // Le contenu réel s'arrête `shortfall` échantillons avant la cible, et ce qui suit + // dans `stretched` n'est que la rallonge de silence étirée. On rejoue en demandant + // une cible plus longue du même montant : la chaîne étant la même, elle en perd + // autant, et le contenu tombe cette fois pile sur `target_samples`. + let corrected_target = target_samples + shortfall + ATEMPO_LENGTH_GUARD; + let corrected_speed = source_samples as f64 / corrected_target as f64; + let corrected_factors = atempo_factors(corrected_speed)?; + let corrected_tail = atempo_prime_tail(&corrected_factors, corrected_speed); + let mut corrected = planes(target_samples); + atempo_pass( + pcm, + &corrected_factors, + corrected_tail, + target_samples, + &mut corrected, + )?; + if corrected[0].len() >= target_samples { + stretched = corrected; + } + } + + // Plus court que la cible : la chaîne n'a pas fait son travail. On rend `None` — compléter + // par des zéros exporterait un trou en se faisant passer pour un succès, et le contrat de + // `stretch_pcm_to_length` est un repli WSOLA sur échec. + if stretched[0].len() < target_samples { + eprintln!( + "[openscreen-compositor] atempo: sortie de {} échantillons pour une cible de {target_samples} (vitesse {speed}, {} étages), repli WSOLA", + stretched[0].len(), + factors.len() + ); + return None; + } + Some(stretched) +} + /// Découpe le PCM gardé avec les mêmes spans et la même quantification frame que la vidéo. pub fn stretch_clip_pcm_by_speed( pcm: &[Vec], @@ -914,6 +1381,153 @@ pub fn assemble_concatenated_pcm( output } +/// Mix imported audio tracks (issue #350) over the assembled programme. +/// +/// Each track is decoded across its trim window — already resampled to 48 kHz +/// stereo by `decode_clip_audio`, the same path a clip's own audio takes — scaled +/// by its per-track gain (the same `10^(dB/20)` law as `finish_audio`), and summed +/// into the programme at `start_sec`. The programme length is NOT extended: a +/// track that runs past the video is truncated to it, so the audio and video +/// streams stay the same length for the muxer. +/// +/// The decode window is capped up front at the room left in the programme after +/// `start_sec`, and a track starting at/after the end is skipped without decoding. +/// `decode_clip_audio` preallocates from the window, so this keeps a long track +/// pinned near a short programme's end from buffering (and clamping away) hours of +/// PCM. `trim_end_sec` must therefore be concrete — the renderer sends +/// `trimEnd ?? durationSec`. +/// +/// A track whose file has no decodable audio is skipped — the same degradation a +/// stream-less clip gets. +pub fn mix_external_tracks(mut programme: PlanarPcm, tracks: &[SceneAudioTrack]) -> PlanarPcm { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if programme_len == 0 { + return programme; + } + for track in tracks { + let offset = (track.start_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; + // A track that starts at or past the programme end contributes nothing — + // skip it before decoding anything. + if offset >= programme_len { + continue; + } + let trim_start = track.trim_start_sec.max(0.0); + let Some(trim_end_full) = track.trim_end_sec else { + // Without a concrete end there is no safe window to decode (see the doc + // comment); the renderer always resolves one, so this only guards a + // hand-written scene. + continue; + }; + // Cap the decode window at the room left in the programme. Everything past + // `offset` that overflows is discarded by `overlay_track_pcm` anyway, so + // decoding it only wastes time and memory — a three-hour track placed at + // second 9 of a ten-second export must not buffer three hours of PCM. + let remaining_sec = (programme_len - offset) as f64 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + let trim_end = trim_end_full.min(trim_start + remaining_sec); + // The track's own length, before that cap. The fades belong to the track, not to + // whatever the programme had room for — capping first and measuring after is what + // made a fade-out ramp down at the truncation point instead of at the real end. + let full_len = + ((trim_end_full - trim_start).max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64) as usize; + if trim_end <= trim_start { + continue; + } + let decoded = match decode_clip_audio(&track.path, trim_start, trim_end) { + Ok(Some(pcm)) => pcm, + _ => continue, + }; + // The app's own range is -60..+12 dB (the inspector slider); clamping at + // -12 here floored every quiet bed at a tenth of the attenuation asked for. + let gain = 10.0f32.powf(track.gain_db.clamp(-60.0, 12.0) / 20.0); + overlay_track_pcm( + &mut programme, + &decoded, + offset, + gain, + track.fade_in_sec.max(0.0), + track.fade_out_sec.max(0.0), + full_len, + ); + } + programme +} + +/// Sum one decoded track into the programme at `offset` samples, scaled by `gain`, +/// truncated at the programme's end. Split out of `mix_external_tracks` so the +/// placement/gain/clamp math is testable without ffmpeg, exactly like +/// `mix_aligned_tracks` is split from the decode above. +fn overlay_track_pcm( + programme: &mut PlanarPcm, + decoded: &PlanarPcm, + offset: usize, + gain: f32, + fade_in_sec: f64, + fade_out_sec: f64, + // The track's length before the programme cap, in samples, or 0 when nothing capped it. + // `decoded` may be shorter because the decode window was capped at the room left in the + // programme; the ramps belong to the track, not to the room. + full_len: usize, +) { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if offset >= programme_len { + return; + } + let room = programme_len - offset; + // The ramps are measured against the DECODED length, not the room left in the + // programme: a track running past the end is cut off there, and a fade-out + // timed to the cut would ramp down over audio the export never reaches. + let decoded_len = decoded.iter().map(Vec::len).max().unwrap_or(0); + let envelope_len = full_len.max(decoded_len); + let (fade_in, fade_out) = resolve_fade_samples(envelope_len, fade_in_sec, fade_out_sec); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let Some(source) = decoded.get(channel) else { + continue; + }; + let count = source.len().min(room); + let dst = &mut programme[channel]; + for k in 0..count { + dst[offset + k] += source[k] * gain * fade_envelope(k, envelope_len, fade_in, fade_out); + } + } +} + +/// Fade lengths in samples, reduced to fit inside `len`. +/// +/// Fades that do not fit share the window in proportion rather than being clamped +/// independently: clamping each to the length first would turn an asymmetric pair +/// into a symmetric one, losing the shape asked for. Kept identical to the app's +/// `resolveFadeSecs` so the preview and the render agree. +fn resolve_fade_samples(len: usize, fade_in_sec: f64, fade_out_sec: f64) -> (usize, usize) { + if len == 0 { + return (0, 0); + } + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let mut fade_in = fade_in_sec.max(0.0) * rate; + let mut fade_out = fade_out_sec.max(0.0) * rate; + let total = fade_in + fade_out; + if total > len as f64 && total > 0.0 { + let scale = len as f64 / total; + fade_in *= scale; + fade_out *= scale; + } + (fade_in.round() as usize, fade_out.round() as usize) +} + +/// Linear ramp factor at sample `k` of a `len`-sample track. +fn fade_envelope(k: usize, len: usize, fade_in: usize, fade_out: usize) -> f32 { + let mut v = 1.0f32; + if fade_in > 0 && k < fade_in { + v = v.min(k as f32 / fade_in as f32); + } + if fade_out > 0 && len > k { + let remaining = len - k; + if remaining <= fade_out { + v = v.min(remaining as f32 / fade_out as f32); + } + } + v +} + /// Encodeur AAC attaché au muxer avant son header. Les paquets utilisent le même interleaver /// que la vidéo ; les pts restent en unités échantillon jusqu'au rescale vers l'AVStream. pub(crate) struct AacEncoder { @@ -1026,6 +1640,40 @@ impl Drop for AacEncoder { } } +#[cfg(test)] +mod hold_tests { + use super::*; + + /// Les images tenues allongent le CRÉNEAU audio du clip sans allonger son PCM, et + /// `assemble_concatenated_pcm` laisse des zéros dans ce qui dépasse. Le silence d'une + /// pause est donc gratuit : aucun fichier muet à décoder, aucune entrée de mix en plus. + #[test] + fn a_longer_slot_than_pcm_leaves_silence_at_its_tail() { + // 2s de créneau à 1 fps, mais seulement 1s de PCM décodé. + let plan = build_audio_concat_plan(&[2], &[true], 1.0); + let one_sec = AUDIO_OUTPUT_SAMPLE_RATE as usize; + let pcm = vec![Some(vec![vec![0.5f32; one_sec]; AUDIO_OUTPUT_CHANNELS])]; + let out = assemble_concatenated_pcm(&pcm, &plan); + assert_eq!(out[0].len(), 2 * one_sec); + assert!((out[0][0] - 0.5).abs() < 1e-6, "le vrai son est bien là"); + assert_eq!(out[0][2 * one_sec - 1], 0.0, "la queue du créneau est du silence"); + } + + /// Et le son réel n'est PAS étiré pour remplir le créneau : la voix garde son rythme. + #[test] + fn the_clips_own_audio_is_not_stretched_to_fill_the_hold() { + let plan = build_audio_concat_plan(&[4], &[true], 1.0); + let one_sec = AUDIO_OUTPUT_SAMPLE_RATE as usize; + let mut source = vec![0.0f32; one_sec]; + source[0] = 1.0; + let pcm = vec![Some(vec![source.clone(), source])]; + let out = assemble_concatenated_pcm(&pcm, &plan); + // L'impulsion reste au premier échantillon, pas répartie sur quatre secondes. + assert!((out[0][0] - 1.0).abs() < 1e-6); + assert_eq!(out[0][1], 0.0); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1036,6 +1684,187 @@ mod tests { vec![samples.to_vec(), samples.to_vec()] } + /// Un sinus 440 Hz de `secs` secondes sur les deux canaux. + fn sine(secs: f64) -> PlanarPcm { + let total = (secs * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; + let mut pcm: PlanarPcm = vec![Vec::with_capacity(total); AUDIO_OUTPUT_CHANNELS]; + for i in 0..total { + let t = i as f32 / AUDIO_OUTPUT_SAMPLE_RATE as f32; + let sample = (2.0 * PI * 440.0 * t).sin() * 0.5; + for channel in 0..AUDIO_OUTPUT_CHANNELS { + pcm[channel].push(sample); + } + } + pcm + } + + /// Hauteur mesurée par passages à zéro montants sur une fenêtre d'une seconde. + fn pitch_hz(plane: &[f32], start: usize) -> usize { + let window = (AUDIO_OUTPUT_SAMPLE_RATE as usize).min(plane.len().saturating_sub(start + 1)); + (start..start + window) + .filter(|&i| plane[i] <= 0.0 && plane[i + 1] > 0.0) + .count() + } + + /// Énergie RMS des `count` derniers échantillons. + fn tail_rms(plane: &[f32], count: usize) -> f32 { + let start = plane.len().saturating_sub(count); + let slice = &plane[start..]; + if slice.is_empty() { + return 0.0; + } + (slice.iter().map(|v| v * v).sum::() / slice.len() as f32).sqrt() + } + + /// Les presets réellement cliquables dans l'éditeur (`SPEED_OPTIONS`), plus les bornes + /// `MIN_PLAYBACK_SPEED` / `MAX_PLAYBACK_SPEED` de `src/components/video-editor/types.ts`. + const EDITOR_SPEEDS: [f64; 13] = [ + 0.1, 0.25, 0.5, 0.75, 1.25, 1.5, 1.75, 2.0, 3.0, 4.0, 5.0, 10.0, 100.0, + ]; + + #[test] + fn atempo_covers_every_editor_speed_without_a_silent_tail() { + // Le bug que ce test verrouille : atempo n'émet jamais sa dernière fenêtre, et + // compléter le manque par des zéros collait jusqu'à 181 ms de blanc (0,1×, quatre + // étages) devant le segment suivant — un dropout audible dans un export « réussi ». + // La rallonge de silence en entrée fait sortir les derniers grains pour de bon, donc + // la fin de région doit porter autant de signal que son milieu, à toute vitesse et + // sur des spans courts comme longs. + for &speed in &EDITOR_SPEEDS { + for &secs in &[0.05f64, 0.5, 3.0] { + let pcm = sine(secs); + let source = pcm[0].len(); + let target = (source as f64 / speed).round() as usize; + if target == 0 { + continue; + } + let stretched = unsafe { avfilter_atempo_stretch(&pcm, target, speed) } + .unwrap_or_else(|| { + panic!("atempo doit couvrir {speed}× sur {secs}s (cible {target})") + }); + for plane in &stretched { + assert_eq!(plane.len(), target, "vitesse {speed}× durée {secs}s"); + } + // 10 ms de queue : le zero-padding d'avant en laissait au moins 5 ms à 0,1×. + // Le trou : un silence numérique en fin de région. Zéro tolérance — la + // correction de tempo est faite pour que le contenu tombe pile sur la cible. + let trailing_silence = + stretched[0].iter().rev().take_while(|v| **v == 0.0).count(); + assert_eq!( + trailing_silence, 0, + "{trailing_silence} échantillons de silence en fin de région à {speed}× sur {secs}s" + ); + let tail = (AUDIO_OUTPUT_SAMPLE_RATE as usize / 100).min(target); + assert!( + tail_rms(&stretched[0], tail) > 0.05, + "queue sans énergie à {speed}× sur {secs}s : rms={}", + tail_rms(&stretched[0], tail) + ); + } + } + } + + #[test] + fn atempo_preserves_pitch_through_a_chain_of_stages() { + // 0,25× et 0,1× sortent des bornes [0.5, 100] d'un seul atempo et passent donc par + // la chaîne multi-étages — le cas que le test d'origine (1,25×, un seul maillon) + // ne touchait pas, alors que 0,25× est un preset de la liste déroulante. + for &speed in &[0.1f64, 0.25, 0.5] { + let pcm = sine(2.0); + let target = (pcm[0].len() as f64 / speed).round() as usize; + let stretched = unsafe { avfilter_atempo_stretch(&pcm, target, speed) } + .unwrap_or_else(|| panic!("la chaîne atempo doit monter à {speed}×")); + let measured = pitch_hz(&stretched[0], target / 2); + assert!( + (measured as f64 - 440.0).abs() <= 2.0, + "hauteur à {speed}× : {measured} Hz (un rééchantillonnage la déplacerait)" + ); + } + } + + #[test] + fn stretch_pcm_to_length_is_exact_at_every_editor_speed() { + // Contrat de bout en bout, repli WSOLA compris : quelle que soit la branche prise, + // la longueur rendue est exactement celle que le plan de concaténation attend. + for &speed in &EDITOR_SPEEDS { + let pcm = sine(0.5); + let target = (pcm[0].len() as f64 / speed).round() as usize; + let stretched = stretch_pcm_to_length(&pcm, target); + assert_eq!(stretched.len(), AUDIO_OUTPUT_CHANNELS); + for plane in &stretched { + assert_eq!(plane.len(), target, "vitesse {speed}×"); + } + } + } + + #[test] + fn wsola_fallback_still_stretches_and_keeps_pitch() { + // Le chemin de repli reste atteignable (avfilter absent d'un build, graphe qui ne + // monte pas) et sa recopie de buffer a été remplacée par un curseur de lecture : + // ce test verrouille qu'il rend toujours la bonne durée à la bonne hauteur. + let pcm = sine(2.0); + let speed = 0.5; + let target = (pcm[0].len() as f64 / speed).round() as usize; + let mut stretcher = WsolaTimeStretcher::new( + AUDIO_OUTPUT_SAMPLE_RATE, + AUDIO_OUTPUT_CHANNELS, + speed, + target, + ); + let mut emitted: PlanarPcm = vec![Vec::new(); AUDIO_OUTPUT_CHANNELS]; + for chunk in [stretcher.push(&pcm), stretcher.flush()] { + for channel in 0..AUDIO_OUTPUT_CHANNELS { + emitted[channel].extend_from_slice(&chunk[channel]); + } + } + // Le WSOLA vise la durée sans la garantir à l'échantillon près : c'est + // `stretch_pcm_to_length` qui recadre. On tolère 1 % ici. + let produced = emitted[0].len() as f64; + assert!( + (produced - target as f64).abs() / (target as f64) < 0.02, + "WSOLA a rendu {produced} pour une cible de {target}" + ); + let measured = pitch_hz(&emitted[0], target / 2); + assert!( + (measured as f64 - 440.0).abs() <= 3.0, + "hauteur WSOLA : {measured} Hz" + ); + } + + #[test] + fn atempo_factors_split_out_of_range_speeds() { + // Dans les bornes : un seul maillon. + assert_eq!(atempo_factors(1.25), Some(vec![1.25])); + assert_eq!(atempo_factors(0.5), Some(vec![0.5])); + // Hors bornes : chaîne dont le produit reconstitue la vitesse. + assert_eq!(atempo_factors(0.2), Some(vec![0.5, 0.5, 0.8])); + assert_eq!(atempo_factors(250.0), Some(vec![100.0, 2.5])); + for speed in [0.07f64, 0.3, 1.0, 3.7, 4_000.0] { + let product: f64 = atempo_factors(speed).expect("dans le plafond").iter().product(); + assert!((product - speed).abs() < 1e-9, "produit={product} attendu={speed}"); + } + // MIN_PLAYBACK_SPEED tient largement dans le plafond. + assert_eq!(atempo_factors(0.1).map(|f| f.len()), Some(4)); + } + + #[test] + fn atempo_declines_a_chain_it_would_have_to_stack() { + // `speed` est `source_samples / target_samples`, pas la vitesse cliquée : une scène + // corrompue où une poignée d'échantillons vise une cible d'une heure produit un + // ratio arbitrairement petit. Sans plafond le chaînage empilait une trentaine + // d'étages — plus d'un millier pour un subnormal — chacun avec sa perte d'amorçage. + assert_eq!(atempo_factors(1.0 / 48_000.0 / 3_600.0), None); + assert_eq!(atempo_factors(f64::MIN_POSITIVE), None); + assert_eq!(atempo_factors(1e30), None); + // Et le repli tient le contrat de longueur : c'est le WSOLA qui prend la main. + let pcm = sine(0.05); + let target = pcm[0].len() * 5_000; + let stretched = stretch_pcm_to_length(&pcm, target); + for plane in &stretched { + assert_eq!(plane.len(), target); + } + } + #[test] fn single_track_passes_through_unchanged() { let track = planar(&[0.25, -0.5, 0.75]); @@ -1044,6 +1873,68 @@ mod tests { assert_eq!(mixed[1], vec![0.25, -0.5, 0.75]); } + // Imported audio track overlay (issue #350). + #[test] + fn overlay_sums_at_offset_with_gain() { + let mut programme = planar(&[0.1, 0.1, 0.1, 0.1]); + // ×2 gain, placed at sample offset 1. + overlay_track_pcm(&mut programme, &planar(&[0.2, 0.2]), 1, 2.0, 0.0, 0.0, 0); + assert_eq!(programme[0], vec![0.1, 0.5, 0.5, 0.1]); + assert_eq!(programme[1], vec![0.1, 0.5, 0.5, 0.1]); + } + + #[test] + fn overlay_truncates_a_track_that_runs_past_the_programme() { + let mut programme = planar(&[0.0, 0.0, 0.0]); + // A 4-sample track placed at offset 2 has room for only 1 sample. + overlay_track_pcm(&mut programme, &planar(&[1.0, 1.0, 1.0, 1.0]), 2, 1.0, 0.0, 0.0, 0); + assert_eq!(programme[0], vec![0.0, 0.0, 1.0]); + } + + #[test] + fn overlay_past_the_end_is_a_no_op() { + let mut programme = planar(&[0.3, 0.3]); + overlay_track_pcm(&mut programme, &planar(&[1.0]), 5, 1.0, 0.0, 0.0, 0); + assert_eq!(programme[0], vec![0.3, 0.3]); + } + + #[test] + fn mix_external_tracks_skips_empty_windows() { + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 0.0, + gain_db: 0.0, + trim_start_sec: 2.0, + trim_end_sec: Some(1.0), // end <= start: empty window, never decoded + fade_in_sec: 0.0, + fade_out_sec: 0.0, + }]; + // The empty window is skipped before any decode, so the programme is + // untouched even though the path does not exist. + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + + #[test] + fn mix_external_tracks_skips_a_track_that_starts_past_the_programme() { + // 2 samples = ~0.00004 s of programme at 48 kHz; the track starts at 1 s, so + // its offset is past the end. It must be skipped before any decode is + // attempted (the path does not exist), never buffering its window. + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 1.0, + gain_db: 0.0, + trim_start_sec: 0.0, + trim_end_sec: Some(3600.0), + fade_in_sec: 0.0, + fade_out_sec: 0.0, + }]; + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + #[test] fn single_track_is_not_clamped() { // Promesse de non-régression : une source mono-piste ressort telle quelle, y compris @@ -1136,6 +2027,91 @@ mod tests { assert!((loud[0][0] - ceiling).abs() < 1e-6); } + #[test] + fn fades_that_fit_are_left_alone() { + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let (fin, fout) = resolve_fade_samples(rate as usize, 0.1, 0.2); + assert_eq!(fin, (0.1 * rate).round() as usize); + assert_eq!(fout, (0.2 * rate).round() as usize); + } + + #[test] + fn fades_too_long_for_the_track_share_it_in_proportion() { + // 6 s + 4 s of fade on a 2 s track → 1.2 s / 0.8 s, not a clamped 1 s / 1 s. + // Mirrors `resolveFadeSecs` on the app side; the two must agree or the + // preview and the render shape the same track differently. + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let len = (2.0 * rate) as usize; + let (fin, fout) = resolve_fade_samples(len, 6.0, 4.0); + assert_eq!(fin, (1.2 * rate).round() as usize); + assert_eq!(fout, (0.8 * rate).round() as usize); + assert!(fin + fout <= len + 1); + } + + #[test] + fn a_fade_in_longer_than_the_track_still_reaches_full_volume() { + // Left unreduced this holds the gain near zero for the whole track — the + // layer exports silent. + let (fin, fout) = resolve_fade_samples(100, 10.0, 0.0); + assert_eq!((fin, fout), (100, 0)); + assert!((fade_envelope(99, 100, fin, fout) - 0.99).abs() < 1e-3); + } + + #[test] + fn the_envelope_ramps_at_both_edges_and_holds_between() { + assert_eq!(fade_envelope(0, 100, 10, 10), 0.0); + assert!((fade_envelope(5, 100, 10, 10) - 0.5).abs() < 1e-6); + assert_eq!(fade_envelope(50, 100, 10, 10), 1.0); + assert!((fade_envelope(95, 100, 10, 10) - 0.5).abs() < 1e-6); + } + + #[test] + fn overlay_applies_the_fade_over_the_decoded_length() { + // The ramps are measured against the DECODED length, not the room left in + // the programme: a fade-out timed to the programme's end would ramp down + // over audio the export never reaches. + let mut programme = planar(&[0.0, 0.0, 0.0, 0.0]); + let decoded = planar(&[1.0, 1.0, 1.0, 1.0]); + // A 4-sample fade-in at 48 kHz is far below one sample of real time, so + // ask for the whole decoded length in seconds. + let four = 4.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + overlay_track_pcm(&mut programme, &decoded, 0, 1.0, four, 0.0, 0); + assert_eq!(programme[0][0], 0.0); + assert!(programme[0][1] > 0.0 && programme[0][1] < 1.0); + assert!(programme[0][3] > programme[0][1]); + } + + #[test] + fn a_capped_track_keeps_its_fade_out_at_its_real_end() { + // The decode window is capped at the room left in the programme, so `decoded` is + // SHORTER than the track. Measuring the ramp against what came back would put the + // fade-out at the truncation point — the export would hear a track fading out that + // is in fact being cut off mid-sentence. + let mut programme = planar(&[0.0, 0.0, 0.0, 0.0]); + let decoded = planar(&[1.0, 1.0, 1.0, 1.0]); + let four = 4.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + // The track really runs eight samples; the programme had room for four. + overlay_track_pcm(&mut programme, &decoded, 0, 1.0, 0.0, four, 8); + // Nothing audible has started to ramp: the fade belongs to samples 4..8, which the + // programme never reaches. + for k in 0..4 { + assert_eq!(programme[0][k], 1.0, "sample {k} should be untouched"); + } + } + + #[test] + fn a_track_gain_below_the_output_bound_is_honoured() { + // The per-track gain range is the inspector's -60..+12, NOT the project + // output trim's ±12: clamping here at -12 floored every quiet bed at a + // tenth of the attenuation asked for. + let mut programme = planar(&[0.0]); + let decoded = planar(&[1.0]); + let gain = 10.0f32.powf(-40.0 / 20.0); + overlay_track_pcm(&mut programme, &decoded, 0, gain, 0.0, 0.0, 0); + assert!((programme[0][0] - gain).abs() < 1e-9); + assert!(programme[0][0] < 10.0f32.powf(-12.0 / 20.0)); + } + #[test] fn output_is_clipped_to_full_scale_and_keeps_its_length() { // The trim can push a hot signal past full scale; the timeline must come back the diff --git a/crates/compositor/src/audio_jobs.rs b/crates/compositor/src/audio_jobs.rs new file mode 100644 index 000000000..8d97be40c --- /dev/null +++ b/crates/compositor/src/audio_jobs.rs @@ -0,0 +1,240 @@ +//! Décodage et étirement de l'audio d'un clip, en parallèle du parcours vidéo. +//! +//! Les trois pipelines faisaient ce travail **dans** le callback `on_clip_end` de +//! `walk_composited_timeline`, donc sur le thread de rendu et entre deux clips. Rien +//! n'appelle `progress()` pendant ce temps : la barre d'export s'arrêtait sur le +//! pourcentage de la dernière frame du clip et y restait pour toute la durée du décodage +//! et de l'étirement. C'est la moitié « reporting » du « figé à ~80 % » — la moitié +//! « coût » a été traitée par le passage à atempo, mais un clip long, un repli WSOLA ou +//! n'importe quelle étape audio future reproduisent le symptôme à l'identique. +//! +//! Y répondre en publiant une progression pendant cette phase aurait demandé de changer le +//! protocole natif → JS (il ne transporte qu'un compteur de frames absolu) et de répartir +//! un total que les deux côtés calculent séparément. Déplacer le travail est plus simple et +//! strictement meilleur : l'audio d'un clip ne dépend que de ce clip, il n'y a donc aucune +//! raison qu'il occupe le thread qui compose les frames du clip suivant. Le parcours vidéo +//! continue de rapporter sa progression sans interruption, et le temps audio disparaît du +//! mur d'export au lieu d'y être seulement mieux affiché — ce que +//! `export-pipeline.md` prétendait déjà. +//! +//! Chaque job ouvre son propre `AVFormatContext` sur le fichier du clip : libavformat +//! n'a pas d'état partagé entre contextes, et le décodeur vidéo du parcours en a un autre +//! sur le même chemin, en lecture seule lui aussi. + +use crate::audio::{decode_clip_audio, stretch_clip_pcm_by_speed, PlanarPcm}; +use crate::regions::SpeedSegment; +use std::collections::VecDeque; +use std::thread::JoinHandle; + +/// Nombre de jobs audio en vol. +/// +/// Un thread par clip serait sans plafond : une timeline de deux cents clips décoderait +/// deux cents pistes à la fois, chacune avec son contexte ffmpeg et son PCM complet en +/// mémoire. Quatre suffisent à couvrir le décodage d'un clip par le rendu du suivant, qui +/// est tout ce qu'on cherche ici. +const MAX_INFLIGHT_AUDIO_JOBS: usize = 4; + +/// Le corps d'un job : décode la fenêtre gardée du clip et l'étire sur ses spans de vitesse. +/// +/// Rend `None` quand le clip se déclare audio mais n'a pas de flux décodable, ou quand le +/// décodage échoue — dans les deux cas l'export continue et le clip sort muet, comme avant +/// que ce travail passe sur un thread. Les deux messages sont les mêmes qu'alors ; ils +/// sortent seulement d'un autre thread. +pub fn decode_and_stretch_clip_audio( + clip_index: usize, + screen_path: &str, + source_start_sec: f64, + source_end_sec: f64, + speed_segments: &[SpeedSegment], + out_fps: f64, +) -> Option { + match decode_clip_audio(screen_path, source_start_sec, source_end_sec) { + Ok(Some(pcm)) => Some(stretch_clip_pcm_by_speed(&pcm, speed_segments, out_fps)), + Ok(None) => { + eprintln!( + "[pipeline] warning: clip #{clip_index} déclaré audio mais sans flux décodable; silence conservé" + ); + None + } + Err(error) => { + eprintln!( + "[pipeline] warning: décodage audio du clip #{clip_index} échoué ({error:#}); silence conservé" + ); + None + } + } +} + +/// Collecte les résultats de jobs indexés lancés au fil du parcours, en bornant le nombre +/// de threads simultanés. +/// +/// L'ordre de restitution est celui des index, pas celui d'achèvement : `into_results` rend +/// un `Vec` de la taille annoncée où chaque case porte le résultat de son clip. +pub struct ClipAudioJobs { + inflight: VecDeque<(usize, JoinHandle)>, + results: Vec>, +} + +impl ClipAudioJobs { + pub fn new(clip_count: usize) -> Self { + Self { + inflight: VecDeque::new(), + results: (0..clip_count).map(|_| None).collect(), + } + } + + /// Lance `job` pour `clip_index`. Si le plafond est atteint, attend d'abord le plus + /// ancien job en vol — celui qui a eu le plus de temps pour finir. + pub fn spawn(&mut self, clip_index: usize, job: impl FnOnce() -> T + Send + 'static) { + while self.inflight.len() >= MAX_INFLIGHT_AUDIO_JOBS { + self.collect_oldest(); + } + self.inflight + .push_back((clip_index, std::thread::spawn(job))); + } + + /// Attend tous les jobs restants et rend les résultats rangés par index de clip. + pub fn into_results(mut self) -> Vec> { + while !self.inflight.is_empty() { + self.collect_oldest(); + } + // `mem::take` et pas un move : le `Drop` ci-dessous interdit de sortir un champ de + // `self`. Il ne trouvera plus rien à joindre, la file étant vide. + std::mem::take(&mut self.results) + } + + fn collect_oldest(&mut self) { + let Some((clip_index, handle)) = self.inflight.pop_front() else { + return; + }; + match handle.join() { + Ok(value) => { + if let Some(slot) = self.results.get_mut(clip_index) { + *slot = Some(value); + } + } + // Un panic dans un job audio ne doit pas emporter l'export : le clip sort + // muet, comme il le faisait déjà quand `decode_clip_audio` échouait. + Err(_) => eprintln!( + "[pipeline] warning: le job audio du clip #{clip_index} a paniqué; silence conservé" + ), + } + } +} + +/// Un `JoinHandle` droppé **détache** son thread. Entre le premier `spawn` et +/// `into_results` il y a des `?` — le parcours lui-même, le flush de l'encodeur — et sur +/// l'un d'eux la collection partait en fumée en laissant jusqu'à quatre décodages en vol +/// dans un addon natif que l'hôte peut décharger. On joint donc à la destruction : rien ne +/// survit à la portée, chemin d'erreur compris. +/// +/// Ce n'est pas une annulation : `decode_clip_audio` est un appel opaque et long, et +/// l'interrompre demanderait de lui passer un `AVIOInterruptCB` — un autre changement, dans +/// un autre fichier. L'attente est bornée par le plus lent des quatre, soit quelques +/// secondes depuis que le stretch passe par atempo, et elle ne coûte que sur un export qui +/// a déjà échoué. +impl Drop for ClipAudioJobs { + fn drop(&mut self) { + for (clip_index, handle) in std::mem::take(&mut self.inflight) { + if handle.join().is_err() { + eprintln!( + "[pipeline] warning: le job audio du clip #{clip_index} a paniqué pendant l'abandon de l'export" + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + #[test] + fn results_are_indexed_by_clip_not_by_completion_order() { + // Le premier job est le plus lent : si on rangeait par ordre d'achèvement, le PCM + // du clip 0 atterrirait sur le clip 2 et l'export monterait l'audio dans le + // désordre sans rien signaler. + let mut jobs = ClipAudioJobs::new(3); + jobs.spawn(0, || { + std::thread::sleep(std::time::Duration::from_millis(60)); + "zero" + }); + jobs.spawn(1, || "one"); + jobs.spawn(2, || "two"); + assert_eq!( + jobs.into_results(), + vec![Some("zero"), Some("one"), Some("two")] + ); + } + + #[test] + fn a_clip_without_a_job_keeps_its_empty_slot() { + // Les clips sans audio ne lancent rien ; leur case doit rester `None` pour que + // `assemble_concatenated_pcm` y mette du silence. + let mut jobs = ClipAudioJobs::new(3); + jobs.spawn(1, || 7u32); + assert_eq!(jobs.into_results(), vec![None, Some(7), None]); + } + + #[test] + fn never_more_than_the_cap_run_at_once() { + // Sans plafond, une timeline longue ouvrirait un contexte ffmpeg et un PCM complet + // par clip, tous en même temps. + let live = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let mut jobs = ClipAudioJobs::new(32); + for index in 0..32 { + let live = Arc::clone(&live); + let peak = Arc::clone(&peak); + jobs.spawn(index, move || { + let now = live.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(5)); + live.fetch_sub(1, Ordering::SeqCst); + index + }); + } + let results = jobs.into_results(); + assert_eq!(results.len(), 32); + assert!(results.iter().enumerate().all(|(i, r)| *r == Some(i))); + assert!( + peak.load(Ordering::SeqCst) <= MAX_INFLIGHT_AUDIO_JOBS, + "jusqu'à {} jobs simultanés pour un plafond de {MAX_INFLIGHT_AUDIO_JOBS}", + peak.load(Ordering::SeqCst) + ); + } + + #[test] + fn dropping_the_collection_joins_its_jobs_instead_of_detaching_them() { + // Le chemin d'erreur : entre le premier `spawn` et `into_results` il y a des `?`. + // Sans le `Drop`, jusqu'à quatre décodages continuaient dans le vide après l'abandon + // de l'export, dans un addon que l'hôte peut décharger. + let finished = Arc::new(AtomicUsize::new(0)); + { + let mut jobs = ClipAudioJobs::new(4); + for index in 0..4 { + let finished = Arc::clone(&finished); + jobs.spawn(index, move || { + std::thread::sleep(std::time::Duration::from_millis(20)); + finished.fetch_add(1, Ordering::SeqCst); + }); + } + // Pas d'`into_results` : on abandonne, comme le ferait un `?`. + } + assert_eq!( + finished.load(Ordering::SeqCst), + 4, + "des jobs tournaient encore après la destruction de la collection" + ); + } + + #[test] + fn a_panicking_job_leaves_its_clip_silent_without_taking_the_export_down() { + let mut jobs = ClipAudioJobs::new(2); + jobs.spawn(0, || panic!("décodage impossible")); + jobs.spawn(1, || 42u32); + assert_eq!(jobs.into_results(), vec![None, Some(42)]); + } +} diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 03a0aa6b9..d27cbdf1a 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -21,6 +21,13 @@ //! par les memes primitives (`draw_layer`) et arrivent par iterations, comme le //! port Metal les a ajoutes -- chacun reutilise `layer.wgsl` (modes deja portes) //! ou une passe dediee (`blur.wgsl`). +//! +//! **Segmentation du sujet webcam.** Les quatre etages tournent ici comme sur les +//! deux autres back-ends : `capture_webcam_rgb` rend la camera dans une cible +//! 256x144 et la relit, `segmentation.rs` (partage, EP CPU d'ONNX Runtime) produit +//! le masque sur son propre thread, `set_webcam_mask` le televerse en R8, et +//! `layer.wgsl` branche dessus sur `fx.z`. Cf. +//! `technical-documentation/engineering/webcam-segmentation.md`. use std::cell::RefCell; @@ -44,12 +51,37 @@ use crate::scene::{Scene, SceneBackground}; const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); +/// Budget du cache de textures image (`img_cache`), en octets. +/// +/// Doit tenir le JEU ACTIF d'une frame -- au pire un wallpaper d'ecran ET un +/// fond de camera, que rien n'empeche d'etre deux 7680x7680 a 225 Mo piece. +/// Sous ce seuil l'eviction ne peut plus rendre de memoire sans toucher au jeu +/// actif, ce qu'elle refuse de faire. 512 Mo borne la fuite (1 774 Mo mesures +/// en parcourant les 18 wallpapers livres) en laissant le jeu actif resident. +const IMG_CACHE_BUDGET_BYTES: u64 = 512 * 1024 * 1024; + /// `&LayerCB` -> `&[u8; 128]`. `LayerCB` est `#[repr(C, align(16))]`, son layout /// EST le buffer uniforme WGSL (16 vec4 + 1 vec2 + 2 f32 = 128 octets). fn layer_bytes(cb: &LayerCB) -> &[u8] { unsafe { std::slice::from_raw_parts(cb as *const LayerCB as *const u8, 128) } } +/// Un calque de fond deja lie, en attente de son `draw`. `_buf`/`_tex`/`_view` +/// ne sont jamais relus : ils gardent en vie ce que le bind group reference +/// jusqu'au submit. Ce backend encode toute la frame avant de la soumettre, la +/// ou D3D11 dessine au fil de l'eau ; d'ou cette boite, la que Windows n'a pas +/// besoin d'equivalent. +/// +/// Vit au niveau module (et non dans `compose_frame`) parce que le fond d'ecran +/// ET le fond de la bulle webcam sont desormais construits par les memes +/// methodes. +struct BgDraw { + _buf: wgpu::Buffer, + _tex: Option, + _view: Option, + bind: wgpu::BindGroup, +} + /// Une copie RT -> staging DEJA SOUMISE, dont le mapping est arme mais pas /// encore recolte. On garde `idx` (l'index de soumission rendu par /// `Queue::submit`) pour n'attendre QUE cette soumission-la, et les dimensions @@ -101,6 +133,112 @@ struct ReadbackRing { pending: std::collections::VecDeque, } +/// Cibles et pipelines de la conversion RGBA -> YUV420P sur le GPU. +/// +/// Trois cibles R8Unorm plutot qu'une seule : Y est en pleine resolution et U/V +/// en demie (4:2:0), et wgpu ne sait pas ecrire des attachements de tailles +/// differentes dans une meme passe. +/// Disposition de la chrominance. PAS un gout : une consequence de l'encodeur +/// qui va consommer la frame. `libopenh264` n'accepte que du YUV420P planaire +/// (ses `pix_fmts` sont yuv420p/yuvj420p), VAAPI encode depuis du NV12. Le +/// compositeur doit donc savoir produire les deux. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum YuvFormat { + /// U et V dans deux plans `R8Unorm` separes. + I420, + /// U et V entrelaces dans un seul plan `Rg8Unorm`. + Nv12, +} + +/// Les cibles de chrominance, dont la forme depend du format. +enum Chroma { + Planar { + _u: wgpu::Texture, + _v: wgpu::Texture, + u_view: wgpu::TextureView, + v_view: wgpu::TextureView, + pipe_u: wgpu::RenderPipeline, + pipe_v: wgpu::RenderPipeline, + }, + Interleaved { + _uv: wgpu::Texture, + uv_view: wgpu::TextureView, + pipe_uv: wgpu::RenderPipeline, + }, +} + +struct YuvTargets { + /// Gardee en vie pour sa vue ; seule la vue sert au rendu. + _y: wgpu::Texture, + y_view: wgpu::TextureView, + chroma: Chroma, + fmt: YuvFormat, + bind: wgpu::BindGroup, + pipe_y: wgpu::RenderPipeline, + /// Dimensions pour lesquelles tout ceci a ete construit : un resize doit + /// tout refaire, et comparer ici est moins fragile que de s'en souvenir. + w: u32, + h: u32, + /// `bytes_per_row` alignes a 256. En 1080p, Y passe de 1920 a 2048 et U/V de + /// 960 a 1024 : contrairement au RGBA (7680 = 30*256, deja aligne), les plans + /// PORTENT du padding, et le lecteur doit le retirer ligne a ligne. + bpr_y: u32, + bpr_uv: u32, + /// Offsets des plans de chrominance dans le buffer de staging unique. + /// Alignes a 256 (exigence de `copy_texture_to_buffer`), ce que la taille du + /// plan Y garantit deja puisque `bpr_y` l'est. En NV12 il n'y a qu'un plan de + /// chrominance : `off_v` vaut alors `off_u` et ne doit pas etre lu. + off_u: u64, + off_v: u64, + total: u64, +} + +// --------------------------------------------------------------------------- +// Segmentation du sujet webcam +// --------------------------------------------------------------------------- + +/// Cadence de l'inference. Meme valeur et meme raison que +/// `compositor_windows::SEGMENTATION_HZ` : une silhouette ne bouge pas de facon +/// perceptible en 16 ms, et c'est le seul levier mesure qui divise le cout par +/// deux sans toucher au modele. +const SEGMENTATION_HZ: u32 = 30; + +/// Cible RGBA + buffer de staging pour extraire la frame webcam a la resolution +/// du modele. Pendant wgpu de `compositor_windows::SegCapture`. +/// +/// La divergence tient au `bpr`. D3D11 rend un row pitch decide par le driver et +/// Metal accepte la largeur nue ; `copy_texture_to_buffer` exige, lui, un +/// `bytes_per_row` multiple de 256. Il est donc padde ICI, a la creation, et +/// depadde a la lecture — exactement ce que `ReadbackRing` fait deja pour le RT. +/// A 256 px de large le padding est nul (1024 est deja aligne), mais rien dans +/// cette structure ne le suppose : c'est `width` qui decide, pas le modele. +struct SegCapture { + /// Cible de la passe de capture, et source de la copie vers `staging`. + rt: wgpu::Texture, + view: wgpu::TextureView, + /// Buffer de staging REUTILISE d'une capture a l'autre : a 30 Hz, en + /// reallouer un par tour serait un cout gratuit. + staging: wgpu::Buffer, + width: u32, + height: u32, + bpr: u32, +} + +/// Texture du masque de segmentation, recreee seulement quand la resolution du +/// modele change — c'est-a-dire jamais, en regime etabli. +/// +/// La vue vit A COTE de la texture plutot que d'etre recreee par draw : +/// `make_bind` lie le binding 4 sur CHAQUE draw de calque (le layout l'exige, cf. +/// `tex_entry(4)`), donc une vue par draw ferait une dizaine d'allocations par +/// frame pour rien. La texture, elle, reste indispensable : `write_texture` +/// prend une `Texture`, pas une `TextureView`. +struct WebcamMask { + tex: wgpu::Texture, + view: wgpu::TextureView, + width: u32, + height: u32, +} + pub struct Compositor { gpu: Gpu, render_w: u32, @@ -142,6 +280,18 @@ pub struct Compositor { /// publiques du compositeur sont `&self`, comme tout le reste de l'etat. readback: RefCell, + /// Conversion RGBA -> Y/U/V sur le GPU, construite a la premiere demande. + /// + /// Paresseuse et non dans `new` pour une raison de contrat : la preview + /// n'en veut pas (elle rend du RGBA a un ``) et la payer a chaque + /// construction de compositeur couterait trois textures et trois pipelines + /// a tout le monde pour le seul benefice de l'export. + yuv: RefCell>, + /// Ring de staging DEDIEE aux plans YUV : ses buffers font 3,1 Mo la ou + /// ceux de `readback` en font 8,3, et melanger les deux tailles dans une + /// seule ring rendrait la reutilisation dependante de l'ordre des appels. + readback_yuv: RefCell, + // Etat pilote par live.rs (interior mutability : les methodes sont `&self`). live_params: RefCell, scene: RefCell>, @@ -155,8 +305,20 @@ pub struct Compositor { text_raster: Option, /// Cache des sprites curseur (PNG RGBA -> texture wgpu), par chemin. Meme - /// role que `img_cache` cote macOS : un sprite chargé une fois par session. - img_cache: RefCell>, + /// role que `img_cache` cote macOS. Charge une fois, PAS pour la session : l'entree + /// est evincable des qu'elle sort du jeu actif d'une frame, et un retour dessus la + /// rechargera -- cf. `cached_image`. + /// Le quatrieme champ du tuple est le tick d'usage, qui donne l'ordre LRU + /// -- cf. `cached_image`. + img_cache: RefCell>, + /// Compteur d'acces de `img_cache`, pour l'ordre LRU. Un compteur plutot + /// que l'index de frame : une frame touche plusieurs entrees, et il faut + /// pouvoir les ordonner entre elles. + img_tick: std::cell::Cell, + /// Valeur de `img_tick` au debut de la frame en cours. Tout ce qui a ete + /// touche depuis appartient au jeu actif et ne peut pas etre evince -- voir + /// `cached_image`. + img_frame_start: std::cell::Cell, /// Copie mipmappee de la frame composee, lue par les annotations « flou » /// (mode 10). `ann_copy` garde la texture en vie, `ann_copy_view` porte tous @@ -172,6 +334,35 @@ pub struct Compositor { /// frame. La longueur de la source sert de temoin de changement, comme cote /// macOS. ann_img_cache: RefCell>, + + // --- Segmentation du sujet webcam (cf. `pump_segmentation`) --- + /// Masque du sujet, R8 a la resolution du modele. Ecrit par + /// `set_webcam_mask`, lu par `make_bind` au moment de construire chaque bind + /// group. `None` tant qu'aucune frame n'a ete segmentee — l'effet reste + /// alors eteint plutot que de rendre une webcam invisible en detourage. + webcam_mask: RefCell>, + /// Cible + staging de la capture, crees a la premiere capture et jamais + /// redimensionnes : le modele a une entree fixe. + seg_capture: RefCell>, + /// Worker d'inference, absent tant que `enable_segmentation` n'a pas ete + /// appele. + seg_worker: RefCell>, + /// Segmenteur tenu SUR LE THREAD DE RENDU, utilise a la place du worker en + /// mode deterministe. Voir `set_segmentation_deterministic`. + seg_sync: RefCell>, + /// Export : cadence par frame et inference synchrone, au lieu de l'horloge + /// et du worker. + seg_deterministic: std::cell::Cell, + /// Boite aux lettres du worker. Le masque est depose depuis le thread + /// d'inference et televerse depuis le thread de rendu : aucun appel wgpu ne + /// traverse de thread, ce qui compte ici puisque `Compositor` n'est ni `Send` + /// ni `Sync` (tout son etat vit dans des `RefCell`). + seg_inbox: std::sync::Arc>>>, + seg_rate: RefCell, + /// Frame RGB reutilisee d'une capture a l'autre. + seg_scratch: RefCell>, + /// Le chargement du modele a echoue : ne pas reessayer a chaque frame. + seg_failed: RefCell, } impl Compositor { @@ -238,6 +429,17 @@ impl Compositor { ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, + // Masque de segmentation du sujet webcam. TOUJOURS declare, meme sans + // masque : wgpu valide le bind group contre le layout, donc une entree + // absente ferait echouer chaque draw et pas seulement ceux qui l'utilisent. + // `dummy_view()` est lie a la place, et la branche du shader n'est de + // toute facon prise que si fx.z > 0.5. + tex_entry(4), + // Plan V. En 5 et pas en 3 : les bindings 0-4 etaient deja + // pris quand le chroma est passe d'un plan entrelace a deux + // plans, et renumeroter aurait touche tous les bind groups + // pour un gain nul. + tex_entry(5), ], }); let pipeline_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { @@ -432,6 +634,13 @@ impl Compositor { free: vec![Self::make_staging(&gpu, readback_bpr, h)], pending: std::collections::VecDeque::new(), }); + // Vide : les buffers YUV sont dimensionnes par `YuvTargets` (qui connait + // les trois `bytes_per_row` alignes) et alloues a la premiere relecture. + let readback_yuv = RefCell::new(ReadbackRing { + depth: 1, + free: Vec::new(), + pending: std::collections::VecDeque::new(), + }); Ok(Compositor { gpu, @@ -454,6 +663,8 @@ impl Compositor { accum_view, readback_bpr, readback, + yuv: RefCell::new(None), + readback_yuv, live_params: RefCell::new(LiveParams::default()), scene: RefCell::new(None), cursor: RefCell::new(None), @@ -461,10 +672,21 @@ impl Compositor { timeline_time: RefCell::new(None), text_raster: crate::text::TextRasterizer::new().ok(), img_cache: RefCell::new(std::collections::HashMap::new()), + img_tick: std::cell::Cell::new(0), + img_frame_start: std::cell::Cell::new(0), ann_copy, ann_copy_view, ann_copy_mips, ann_img_cache: RefCell::new(std::collections::HashMap::new()), + webcam_mask: RefCell::new(None), + seg_capture: RefCell::new(None), + seg_worker: RefCell::new(None), + seg_sync: RefCell::new(None), + seg_deterministic: std::cell::Cell::new(false), + seg_inbox: std::sync::Arc::new(std::sync::Mutex::new(None)), + seg_rate: RefCell::new(crate::segmentation::RateLimiter::new(SEGMENTATION_HZ)), + seg_scratch: RefCell::new(Vec::new()), + seg_failed: RefCell::new(false), }) } @@ -773,7 +995,7 @@ impl Compositor { unsafe fn nv12_srvs( &self, frame: *const AVFrame, - ) -> Result<(wgpu::TextureView, wgpu::TextureView)> { + ) -> Result<(wgpu::TextureView, wgpu::TextureView, wgpu::TextureView)> { crate::linux_frames::nv12_planes(frame) } @@ -916,7 +1138,7 @@ impl Compositor { fn make_bind( &self, cb: &LayerCB, - planes: Option<(&wgpu::TextureView, &wgpu::TextureView)>, + planes: Option<(&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView)>, dummy: &wgpu::TextureView, ) -> (wgpu::Buffer, wgpu::BindGroup) { let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { @@ -924,7 +1146,16 @@ impl Compositor { contents: layer_bytes(cb), usage: wgpu::BufferUsages::UNIFORM, }); - let (y, uv) = planes.unwrap_or((dummy, dummy)); + let (y, u, v) = planes.unwrap_or((dummy, dummy, dummy)); + // Le masque est lie sur TOUS les draws, pas seulement celui de la camera. + // wgpu valide le bind group contre le layout : le binding 4 est declare + // (`tex_entry(4)`), donc une entree absente ferait echouer CHAQUE draw et + // pas seulement ceux qui l'echantillonnent. Le lier partout ne coute rien + // — la branche du shader n'est prise que si `fx.z > 0.5`, et seul le + // calque webcam leve `fx.z`. `dummy` reste le repli tant qu'aucune frame + // n'a ete segmentee. + let mask = self.webcam_mask.borrow(); + let mask_view = mask.as_ref().map_or(dummy, |m| &m.view); let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("layer"), layout: &self.bind_group_layout, @@ -939,12 +1170,20 @@ impl Compositor { }, wgpu::BindGroupEntry { binding: 2, - resource: wgpu::BindingResource::TextureView(uv), + resource: wgpu::BindingResource::TextureView(u), }, wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::Sampler(&self.sampler), }, + wgpu::BindGroupEntry { + binding: 4, + resource: wgpu::BindingResource::TextureView(mask_view), + }, + wgpu::BindGroupEntry { + binding: 5, + resource: wgpu::BindingResource::TextureView(v), + }, ], }); (uniform, bind) @@ -993,6 +1232,599 @@ impl Compositor { Ok((tex, w, h)) } + /// Ouvre une frame du point de vue d'`img_cache` : tout ce qui sera touche + /// apres cet appel est le jeu actif, et devient inevincable jusqu'a la + /// frame suivante. + fn begin_image_frame(&self) { + // `+ 1` : la premiere entrée de cette frame recevra `img_tick + 1`, et la protection + // porte sur `tick >= img_frame_start`. Sans le decalage on protégerait aussi la + // DERNIERE entrée de la frame precedente, qui n'appartient plus au jeu actif — le + // résident pourrait alors dépasser le budget d'une texture entière. + self.img_frame_start.set(self.img_tick.get() + 1); + } + + /// Texture d'un fichier image, decodee une seule fois puis reutilisee. + /// + /// Le cache etait NON BORNE, et c'est un vrai cout : les wallpapers livres + /// pesent 23,7 Mo sur disque mais 1 774 Mo une fois decodes en RGBA8 -- + /// `wallpaper8.jpg` fait 7680x7680, soit 225 Mo a lui seul. Parcourir le + /// selecteur les chargeait tous et n'en liberait aucun. + /// + /// L'eviction est LRU sous un budget en octets, et ne touche jamais une + /// texture que la frame EN COURS a deja servie : sans ca, un fond d'ecran + /// et un fond de camera un peu gros se chasseraient l'un l'autre a chaque + /// frame, et un decodage coute 129 ms contre les ~3,5 ms d'une frame. Si le + /// jeu actif depasse a lui seul le budget, on depasse le budget. + fn cached_image(&self, path: &str) -> Result<(wgpu::Texture, u32, u32)> { + let tick = self.img_tick.get() + 1; + self.img_tick.set(tick); + // Recherche isolee dans un `let` pour que l'emprunt immuable soit + // relache AVANT le `borrow_mut` (piege du double emprunt 1re frame). + let hit = self.img_cache.borrow().get(path).cloned(); + if let Some((tex, w, h, _)) = hit { + self.img_cache.borrow_mut().insert(path.to_string(), (tex.clone(), w, h, tick)); + return Ok((tex, w, h)); + } + let (tex, w, h) = self.load_image_texture(path)?; + let mut cache = self.img_cache.borrow_mut(); + cache.insert(path.to_string(), (tex.clone(), w, h, tick)); + // La politique vit dans `frame_geometry` : les trois backends la + // partagent, comme la geometrie, plutot que d'entretenir trois copies + // qui finiraient par diverger. + let entries: Vec<(String, u64, u64)> = + cache.iter().map(|(k, e)| (k.clone(), e.1 as u64 * e.2 as u64 * 4, e.3)).collect(); + let protect_from = self.img_frame_start.get(); + for key in + crate::frame_geometry::lru_evictions(&entries, IMG_CACHE_BUDGET_BYTES, protect_from) + { + cache.remove(&key); + } + Ok((tex, w, h)) + } + + /// Calque image (mode 6) couvrant `dst`, en cover-fit contre `aspect` -- le + /// ratio du RECT vise, et non celui de la sortie : le rognage se calcule + /// contre la zone qu'on remplit, ce qui permet a la bulle webcam d'emprunter + /// le chemin du fond d'ecran au lieu d'en refaire un. + /// + /// Err plutot qu'un repli maison : chaque appelant a son propre message et + /// son propre repli, et un echec silencieux redonnerait le noir qu'on corrige. + fn image_bg_draw( + &self, + path: &str, + dst: [f32; 4], + quad_px: [f32; 2], + radius_px: f32, + aspect: f32, + dummy: &wgpu::TextureView, + ) -> Result { + let (tex, iw, ih) = self.cached_image(path)?; + // Cover-fit : l'image remplit tout le rect, on rogne l'axe long. + let ai = iw as f32 / ih.max(1) as f32; + let src = if ai > aspect { + let vis = aspect / ai; + [(1.0 - vis) * 0.5, 0.0, 1.0 - (1.0 - vis) * 0.5, 1.0] + } else { + let vis = ai / aspect; + [0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5] + }; + let cb = LayerCB { + dst, + src, + quad_px, + radius_px, + mode: 6.0, + ..Default::default() + }; + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), dummy); + Ok(BgDraw { _buf: buf, _tex: Some(tex), _view: Some(view), bind }) + } + + /// Prepare le fond du mode « personnalise », peint DANS la bulle webcam juste + /// avant que la camera n'y soit decoupee par-dessus. + /// + /// Le shader ne sait peindre qu'une couleur plate sous le masque, donc un + /// degrade ou une image y tombaient sur du noir -- et le defaut EST une image + /// (`DEFAULT_WALLPAPER`), si bien que le mode ne rendait jamais ce que le + /// selecteur montrait. Peindre le fond puis composer la camera en detourage + /// donne exactement le meme resultat (`lerp(fond, camera, personne)`, ici par + /// le melange alpha) pour les trois sortes de fond, en reutilisant les chemins + /// deja eprouves du fond d'ecran, et sans rien ajouter aux trois shaders. + /// + /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit epouser ses + /// coins arrondis, sinon un rectangle deborde derriere la camera. + fn webcam_bg_draw( + &self, + bg: Option<&SceneBackground>, + dst: [f32; 4], + quad_px: [f32; 2], + radius_px: f32, + dummy: &wgpu::TextureView, + ) -> BgDraw { + const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + let flat = |cb: LayerCB| { + let (buf, bind) = self.make_bind(&cb, None, dummy); + BgDraw { _buf: buf, _tex: None, _view: None, bind } + }; + let solid = |color: [f32; 4]| LayerCB { + dst, + quad_px, + radius_px, + mode: 1.0, + color, + ..Default::default() + }; + match bg { + Some(SceneBackground::Color { color }) => { + flat(solid(parse_hex(color).unwrap_or(BLACK))) + } + Some(SceneBackground::Gradient { angle_deg, stops }) => { + let c0 = stops.first().and_then(|s| parse_hex(s)).unwrap_or(BLACK); + let c1 = stops.last().and_then(|s| parse_hex(s)).unwrap_or(c0); + // angle CSS -> direction unitaire, meme convention que le fond + // d'ecran (dont la direction se lit en espace SORTIE : le degrade + // traverse le cadre, la bulle n'en montre que sa tranche). + let a = angle_deg.to_radians(); + flat(LayerCB { + dst, + src: [c1[0], c1[1], c1[2], c1[3]], + quad_px, + radius_px, + mode: 5.0, + color: c0, + fx: [a.sin(), -a.cos(), 0.0, 0.0], + ..Default::default() + }) + } + Some(SceneBackground::Image { path }) => { + // Le cover-fit se mesure sur la BULLE, pas sur la sortie : c'est + // elle que l'image doit remplir sans etirement. + let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; + match self.image_bg_draw(path, dst, quad_px, radius_px, aspect, dummy) { + Ok(d) => d, + Err(e) => { + // Meme contrat que le fond d'ecran : un chemin casse est + // logge puis remplace par du noir. Un repli silencieux + // redonnerait le bug qu'on corrige. + eprintln!("[fond webcam] \"{path}\" : {e:#}"); + flat(solid(BLACK)) + } + } + } + // Personnalise sans fond : noir, comme avant -- mais c'est desormais + // le seul chemin qui y mene, au lieu de l'etre pour toute image et + // tout degrade. + None => flat(solid(BLACK)), + } + } + + // -- segmentation du sujet webcam -- + + /// Extrait la frame webcam en RGB8 a la resolution du modele, dans `out`. + /// + /// Pendant wgpu de `compositor_windows::capture_webcam_rgb`, avec les memes + /// contraintes d'appel. Comme cote Metal, rien n'est « requisitionne » : la + /// passe s'ouvre sur `SegCapture::view` et se referme. La contrainte d'ordre + /// tient malgre tout, et pour une autre raison — cette methode ATTEND sa + /// propre soumission, donc l'appeler une fois la passe de composition + /// encodee serialiserait CPU et GPU sur exactement le chemin que cette + /// conception garde recouvert. Elle tourne donc dans le prologue de + /// `compose_frame`, avant le moindre encodeur. + /// + /// `src` est le rect source en UV. L'appelant y passe la frame ENTIERE et non + /// le sous-rect dessine — cf. `pump_segmentation`. + /// + /// # Le readback + /// + /// Meme forme que `ReadbackRing`, en plus simple parce qu'il n'y a rien a + /// recouvrir : une seule copie, attendue tout de suite. Ce qui EST repris de + /// la ring, parce que c'est la lecon qu'elle porte, c'est + /// `WaitForSubmissionIndex` — jamais `Maintain::Wait`, qui absorberait toute + /// la file GPU (3,8 a 6,2 ms mesurees en 1080p, cf. l'en-tete de + /// `ReadbackRing`) au lieu de la seule copie de 147 Ko demandee ici. + /// + /// C'est le second readback synchrone du chemin de preview, qui en paie deja + /// un a profondeur 1 (`live.rs`). C'est le seul cout que ce portage ajoute au + /// rendu, et il ne se paie que quand un effet est demande. + /// + /// A l'EXPORT, ou la ring tourne a profondeur 2, il faut etre honnete sur ce + /// que cette attente coute : une file GPU se termine dans l'ordre, donc + /// attendre CETTE soumission, c'est attendre aussi la copie de la frame + /// precedente que la ring gardait justement en vol. Le travail CPU de la + /// frame courante ne la recouvre donc plus. Ce n'est pas gratuit, c'est + /// seulement borne : 30 Hz et non 60, et zero quand aucun effet n'est demande. + /// Aucune des deux mesures §C.2 n'a ete faite — cf. « Still open » dans + /// `webcam-segmentation.md`. + pub unsafe fn capture_webcam_rgb( + &self, + wy: &wgpu::TextureView, + wu: &wgpu::TextureView, + wv: &wgpu::TextureView, + src: [f32; 4], + width: u32, + height: u32, + out: &mut Vec, + ) -> Result<()> { + if width == 0 || height == 0 { + anyhow::bail!("capture webcam de dimensions nulles ({width}x{height})"); + } + { + let mut slot = self.seg_capture.borrow_mut(); + if !matches!(slot.as_ref(), Some(c) if c.width == width && c.height == height) { + let rt = self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("seg-capture"), + size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + // Meme format que le RT : c'est celui que `mk_layer` a cable + // dans la cible couleur du pipeline de calque, et une passe + // dont la piece jointe ne l'a pas est refusee. + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let view = rt.create_view(&wgpu::TextureViewDescriptor::default()); + let bpr = (width * 4).div_ceil(256) * 256; + let staging = self.gpu.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("seg-capture-staging"), + size: u64::from(bpr) * u64::from(height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + *slot = Some(SegCapture { rt, view, staging, width, height, bpr }); + } + } + let slot = self.seg_capture.borrow(); + let cap = slot.as_ref().expect("cree juste au-dessus"); + + // Plein cadre de la cible, sans coins ni motion blur : le modele veut + // l'image, pas la mise en forme. `fx` reste a zero — la branche de masque + // du shader ne doit surtout pas se prendre sur la capture qui l'alimente. + // `color.a = 1` n'est pas decoratif : `fs_main` calcule son alpha en + // `layer.color.a * alpha_mask`, donc le defaut (0) rendrait un quad + // entierement transparent. + let (_uniform, bind) = self.make_bind( + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src, + quad_px: [width as f32, height as f32], + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + mb: [1.0, 1.0, 1.0, 0.0], + ..Default::default() + }, + Some((wy, wu, wv)), + &self.dummy_view(), + ); + + let mut encoder = self.gpu.device.create_command_encoder( + &wgpu::CommandEncoderDescriptor { label: Some("seg-capture") }, + ); + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("seg-capture-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &cap.view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + rpass.set_bind_group(0, &bind, &[]); + rpass.draw(0..4, 0..1); + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &cap.rt, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &cap.staging, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(cap.bpr), + rows_per_image: Some(height), + }, + }, + wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + ); + let idx = self.gpu.context.submit(std::iter::once(encoder.finish())); + let (tx, rx) = std::sync::mpsc::channel(); + cap.staging.slice(..).map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + // `WaitForSubmissionIndex` et JAMAIS `Maintain::Wait` : cf. l'en-tete de + // `ReadbackRing`, qui est le proces-verbal de cette regression-la. + self.gpu.device.poll(wgpu::Maintain::WaitForSubmissionIndex(idx)); + rx.recv() + .map_err(|_| anyhow::anyhow!("map_async channel (capture webcam)"))? + .map_err(|e| anyhow::anyhow!("map_async (capture webcam): {e:?}"))?; + let slice = cap.staging.slice(..); + let mapped = slice.get_mapped_range(); + + let (w, h, bpr) = (width as usize, height as usize, cap.bpr as usize); + // `clear` + `reserve` plutot qu'un `Vec` neuf : la capacite survit d'une + // capture a l'autre, donc apres le premier tour plus une seule + // reallocation. A 30 Hz ce n'est pas une coquetterie. + out.clear(); + out.reserve(w * h * 3); + for row in 0..h { + // La ligne fait `w * 4` octets utiles dans un pas de `bpr` : le + // padding d'alignement se saute ici, il n'a jamais de sens pour le + // modele. + for px in mapped[row * bpr..row * bpr + w * 4].chunks_exact(4) { + // RGBA -> RGB : le modele n'a pas de canal alpha en entree. + out.push(px[0]); + out.push(px[1]); + out.push(px[2]); + } + } + drop(mapped); + // Sans `unmap`, la capture suivante echouerait a re-armer `map_async` sur + // un buffer deja mappe. + cap.staging.unmap(); + Ok(()) + } + + /// Publie le masque de segmentation du sujet webcam (R8, `width`x`height`, + /// 0 = fond). + /// + /// `Queue::write_texture` et non une copie par buffer : il n'impose aucun + /// alignement de ligne (c'est `copy_texture_to_buffer` qui exige 256, cf. + /// `SegCapture`), et c'est deja par lui que `linux_frames` televerse les plans + /// NV12 avec les strides SIMD de swscale. La texture n'est recreee que si la + /// resolution du modele change, ce qui n'arrive pas en regime etabli. + /// + /// Pas de double buffer, et pour la meme raison que cote Metal : quand + /// `pump_segmentation` appelle ceci, la frame precedente est deja drainee — + /// `capture_webcam_rgb` attend sa soumission, et la preview comme l'export + /// passent par `readback_take`, qui attend la sienne. Si cet invariant + /// changeait, c'est ce code-ci qui casserait. + pub fn set_webcam_mask(&self, data: &[u8], width: u32, height: u32) -> Result<()> { + if width == 0 || height == 0 { + anyhow::bail!("masque webcam de dimensions nulles ({width}x{height})"); + } + let expected = (width as usize) * (height as usize); + if data.len() < expected { + anyhow::bail!( + "masque webcam trop court : {} octets pour {width}x{height}", + data.len() + ); + } + + let mut slot = self.webcam_mask.borrow_mut(); + if !matches!(slot.as_ref(), Some(m) if m.width == width && m.height == height) { + let tex = self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("webcam-mask"), + size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + *slot = Some(WebcamMask { tex, view, width, height }); + } + let mask = slot.as_ref().expect("alloue juste au-dessus"); + self.gpu.context.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &mask.tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + // `data` peut etre plus long que le masque (le garde ci-dessus est un + // minimum) : on ne televerse que ce que la texture porte. + &data[..expected], + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(width), + rows_per_image: Some(height), + }, + wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + ); + Ok(()) + } + + /// Un tour de segmentation : televerse le masque pret, puis soumet une + /// nouvelle frame si la cadence l'autorise. Port de + /// `compositor_windows::pump_segmentation` — worker, boite aux lettres, + /// limiteur de cadence et demarrage paresseux sont independants de la + /// plateforme, seuls les deux appels GPU changent. + /// + /// Les deux moities sont volontairement desynchronisees. Le masque televerse + /// ici vient de la frame precedente — une frame de retard sur une silhouette + /// est invisible, alors qu'attendre l'inference bloquerait le rendu, ce qui + /// est exactement le cout que toute cette conception cherche a ne pas payer. + unsafe fn pump_segmentation( + &self, + wy: &wgpu::TextureView, + wu: &wgpu::TextureView, + wv: &wgpu::TextureView, + valid: [f32; 2], + ) -> Result<()> { + if *self.seg_failed.borrow() { + return Ok(()); + } + // Rien a faire si aucun effet n'est demande : ni capture, ni inference, + // ni masque. Le cout de la fonctionnalite est alors exactement nul. + let (wants_effect, model_path) = { + let scene = self.scene.borrow(); + match scene.as_ref().and_then(|s| s.webcam_effect.as_ref()) { + Some(e) if e.shader_code() > 0.0 => (true, e.model_path.clone()), + _ => (false, None), + } + }; + if !wants_effect { + return Ok(()); + } + + // Demarrage paresseux, pilote par la scene : personne n'a a appeler + // `enable_segmentation` a la main, et un modele introuvable eteint l'effet + // au lieu de faire tomber le rendu. + if self.seg_worker.borrow().is_none() && self.seg_sync.borrow().is_none() { + let Some(path) = model_path else { return Ok(()) }; + if let Err(e) = self.enable_segmentation(std::path::Path::new(&path)) { + eprintln!("[segmentation] desactivee : {e}"); + // Une scene qui reste identique retenterait a chaque frame ; on + // leve le verrou plutot que de journaliser 60 fois par seconde. + *self.seg_failed.borrow_mut() = true; + return Ok(()); + } + // En preview on rend cette frame sans masque : le worker vient de + // demarrer et l'effet apparaitra dans quelques millisecondes, ce que + // personne ne voit. A l'export cette frame part dans le fichier — on + // enchaine donc sur la capture et l'inference plutot que de la laisser + // sortir non detouree. + if !self.seg_deterministic.get() { + return Ok(()); + } + } + + if let Some(mask) = self.seg_inbox.lock().unwrap().take() { + self.set_webcam_mask( + &mask, + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + )?; + } + + // La cadence horloge est le bon reglage en preview et le mauvais a + // l'export, ou les frames defilent aussi vite que la machine decode : le + // nombre de frames couvertes par un masque dependrait alors de la charge. + // En deterministe, une inference par frame. + if !self.seg_deterministic.get() + && !self.seg_rate.borrow_mut().should_run(std::time::Instant::now()) + { + return Ok(()); + } + let mut scratch = self.seg_scratch.borrow_mut(); + // La frame ENTIERE, pas le sous-rect dessine : un crop utilisateur serre + // amputerait le sujet en entree du modele, et le masque serait faux la ou + // il compte le plus. Le shader ramene ses coordonnees dans cet espace via + // `fx.xy`. + self.capture_webcam_rgb( + wy, + wu, + wv, + [0.0, 0.0, valid[0], valid[1]], + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + &mut scratch, + )?; + if self.seg_deterministic.get() { + // Synchrone : le masque doit exister avant que cette frame ne soit + // composee, sinon on retombe sur le defaut qu'on corrige. Une + // inference ratee laisse le masque precedent, comme le fait le worker. + let mut sync = self.seg_sync.borrow_mut(); + if let Some(seg) = sync.as_mut() { + match seg.run(&scratch) { + Ok(mask) => { + // `run` rend une tranche empruntee au segmenteur : copier + // puis relacher, sinon `set_webcam_mask` reemprunterait + // `seg_sync` encore emprunte ici. + let mask = mask.to_vec(); + drop(sync); + self.set_webcam_mask( + &mask, + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + )?; + } + Err(e) => eprintln!("[segmentation] frame ignoree : {e}"), + } + } + } else if let Some(w) = self.seg_worker.borrow().as_ref() { + w.submit(&scratch); + } + Ok(()) + } + + /// Demarre la segmentation du sujet webcam pour ce compositeur. + /// + /// Idempotent. Tant qu'elle n'est pas appelee, `compose_frame` ne fait rien de + /// plus et la webcam se dessine comme avant — c'est ce qui rend l'effet inerte + /// plutot que casse sur une build sans modele. + pub fn enable_segmentation(&self, model_path: &std::path::Path) -> Result<()> { + if self.seg_worker.borrow().is_some() || self.seg_sync.borrow().is_some() { + return Ok(()); + } + let segmenter = crate::segmentation::Segmenter::load(model_path)?; + // En deterministe, le segmenteur reste ici : l'inference tourne sur le + // thread de rendu, donc le masque de la frame N est pret AVANT qu'elle ne + // soit composee. Le worker est un choix de preview — ne jamais bloquer + // l'affichage — et c'est exactement ce qui rend l'export irreproductible, + // le masque arrivant quelques frames plus tard selon la charge. + if self.seg_deterministic.get() { + *self.seg_sync.borrow_mut() = Some(segmenter); + return Ok(()); + } + let inbox = std::sync::Arc::clone(&self.seg_inbox); + let worker = + crate::segmentation::SegmentationWorker::spawn(segmenter, move |mask, _, _| { + // Ecrase le masque precedent s'il n'a pas encore ete televerse : + // c'est le plus recent qui vaut, jamais une file. + *inbox.lock().unwrap() = Some(mask.to_vec()); + }); + *self.seg_worker.borrow_mut() = Some(worker); + Ok(()) + } + + /// Bascule la segmentation en mode reproductible, pour l'export. + /// + /// En preview, la cadence suit l'horloge (30 Hz reels) et l'inference tourne + /// sur un worker : c'est le bon choix, l'affichage ne doit jamais attendre. A + /// l'export les frames sont rendues aussi vite que la machine decode, sans + /// rapport avec le temps reel — et ces deux choix deviennent alors des bugs. + /// La cadence horloge fait dependre le nombre de frames couvertes par un + /// masque de la vitesse de la machine, et le worker asynchrone rend les + /// premieres frames AVANT que le premier masque n'existe : elles partent dans + /// le fichier avec le vrai arriere-plan de la webcam. Deux exports du meme + /// projet ne donnent donc pas les memes pixels, ce qui casse l'invariant + /// « l'export est identique a la preview ». + /// + /// En deterministe : une inference PAR FRAME, synchrone. Plus couteux + /// (~3 ms/frame), mais l'export est hors ligne et chaque frame porte le masque + /// calcule depuis SA propre image. + /// + /// A appeler avant la premiere frame — c'est ce qui decide comment + /// `enable_segmentation` s'installe. + pub fn set_segmentation_deterministic(&self, on: bool) { + if self.seg_deterministic.get() == on { + return; + } + self.seg_deterministic.set(on); + // Changer de mode change le MOTEUR, et `enable_segmentation` est idempotent sur la + // PRESENCE d'un moteur : sans demonter celui qui ne correspond plus, le drapeau + // mentirait. Un compositeur qui a deja servi en preview garderait son worker, + // `seg_sync` resterait vide, et l'export entier ne ferait AUCUNE inference. Le + // demarrage paresseux de `pump_segmentation` reinstalle le bon moteur a la frame + // suivante. + *self.seg_worker.borrow_mut() = None; + *self.seg_sync.borrow_mut() = None; + // Et le masque que le worker demonte avait peut-etre deja depose : il vient de l'autre + // mode, il n'a rien a faire sur la premiere frame de celui-ci. + *self.seg_inbox.lock().unwrap() = None; + } + + /// Eteint l'effet : la webcam se redessine telle quelle a la frame suivante. + pub fn clear_webcam_mask(&self) { + *self.webcam_mask.borrow_mut() = None; + } + /// Rend une frame dans le RT interne. Le screen `screen`/`webcam` sont des /// carriers `linux_frames` ; la geometrie vient de `plan_frame`. Coeur : /// fond uni + ecran cover-fit. `readback_direct` lit ensuite le RT. @@ -1003,10 +1835,11 @@ impl Compositor { frame: f32, cfg: &Cfg, ) -> Result<()> { + self.begin_image_frame(); if Self::pixel_buffer_of(screen).is_none() { return self.clear_rt(); } - let (sy, suv) = self.nv12_srvs(screen)?; + let (sy, su, sv) = self.nv12_srvs(screen)?; let (stw, sth) = self.tex_dims(screen); let (wtw, wth) = self.tex_dims(webcam); let (scw, sch) = ((*screen).width as f32, (*screen).height as f32); @@ -1018,6 +1851,36 @@ impl Compositor { let u_max = scw / (stw.max(1)) as f32; let v_max = sch / (sth.max(1)) as f32; let (rw, rh) = (self.render_w as f32, self.render_h as f32); + // Etendue valide de la texture webcam : les decodeurs allouent des + // textures alignees (`linux_frames` arrondit deja aux dimensions paires), + // donc la frame n'occupe pas forcement toute la texture. `.max(1)` au + // denominateur — `tex_dims` rend (1, 1) sur une webcam absente, la ou le + // chemin Windows divise sans garde parce qu'il a toujours les deux frames. + let w_valid = [wcw / (wtw.max(1)) as f32, wch / (wth.max(1)) as f32]; + + // Segmentation, AVANT le moindre encodeur de composition : + // `capture_webcam_rgb` attend sa propre soumission, et attendre au milieu + // de la frame serialiserait CPU et GPU. Dernier point ou `wtw/wth/wcw/wch` + // sont en portee sans emprunt de `self.scene` — `pump_segmentation` + // emprunte la scene lui-meme. + // + // L'effet est teste ICI en plus de l'etre dans `pump_segmentation` : sur + // ce backend `nv12_srvs` ALLOUE deux `TextureView` a chaque appel (il n'y + // a pas de cache, cf. `clear_srv_cache`), et la fonctionnalite doit couter + // exactement zero quand elle est eteinte — ce qui est le cas general. + let wants_seg = self + .scene + .borrow() + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .is_some_and(|e| e.shader_code() > 0.0); + if wants_seg && !webcam.is_null() { + // `nv12_srvs` dereference `data[0]` sans verifier la frame elle-meme, + // d'ou le garde de nullite au-dessus (meme condition que le draw PiP). + if let Ok((wy, wu, wv)) = self.nv12_srvs(webcam) { + self.pump_segmentation(&wy, &wu, &wv, w_valid)?; + } + } let scene_ref = self.scene.borrow(); let cursor_ref = self.cursor.borrow(); @@ -1113,7 +1976,7 @@ impl Compositor { color: [1.0, 1.0, 1.0, 1.0], src_prev: g.cut, dst_prev: g.s_dst_prev, - mb: [g.mb_taps, 1.0, 1.0, 0.0], + mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], ..Default::default() }, Some(quad) => self.tilted_screen_cb(quad, s_px, quad_center_px, g.cut, g.s_radius), @@ -1122,7 +1985,7 @@ impl Compositor { // `_screen_uniform` garde le buffer uniforme en vie (reference par le bind). let dummy = self.dummy_view(); let (_screen_uniform, screen_bind) = - self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); + self.make_bind(&screen_layer, Some((&sy, &su, &sv)), &dummy); // OMBRE PORTEE de l'ecran, dessinee JUSTE AVANT le calque ecran. Le shader // la connait depuis le debut ; ce qui manquait etait uniquement le draw @@ -1154,54 +2017,23 @@ impl Compositor { }); // Fond (gradient mode 5 OU image mode 6), dessine dans la passe de fond. - // `_tex`/`_view` gardent l'image en vie pendant le pass. - struct BgDraw { - _buf: wgpu::Buffer, - _tex: Option, - _view: Option, - bind: wgpu::BindGroup, - } let bg_draw = bg_layer.and_then(|bl| match bl { BgLayer::Gradient(cb) => { let (buf, bind) = self.make_bind(&cb, None, &dummy); Some(BgDraw { _buf: buf, _tex: None, _view: None, bind }) } + // Le wallpaper couvre tout le cadre, donc dst plein et pas de coins : + // `image_bg_draw` sert aussi la bulle webcam, qui elle en a. BgLayer::Image(path) => { - // Charge (ou recupere du cache) le wallpaper. Emprunt isole AVANT - // le borrow_mut (piege du double emprunt 1re frame, cf. macOS). - let cached = self.img_cache.borrow().get(path.as_str()).cloned(); - let (tex, iw, ih) = match cached { - Some(v) => v, - None => match self.load_image_texture(&path) { - Ok(v) => { - self.img_cache.borrow_mut().insert(path.clone(), v.clone()); - v - } - Err(e) => { - eprintln!("[fond image] \"{path}\" : {e:#}"); - return None; - } - }, - }; - // Cover-fit : l'image remplit tout le cadre, on rogne l'axe long. - let ai = iw as f32 / ih.max(1) as f32; - let ao = rw / rh; - let src = if ai > ao { - let vis = ao / ai; - [(1.0 - vis) * 0.5, 0.0, 1.0 - (1.0 - vis) * 0.5, 1.0] - } else { - let vis = ai / ao; - [0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5] - }; - let cb = LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - src, - mode: 6.0, - ..Default::default() - }; - let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); - let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); - Some(BgDraw { _buf: buf, _tex: Some(tex), _view: Some(view), bind }) + match self + .image_bg_draw(&path, [0.0, 0.0, 1.0, 1.0], [0.0, 0.0], 0.0, rw / rh, &dummy) + { + Ok(d) => Some(d), + Err(e) => { + eprintln!("[fond image] \"{path}\" : {e:#}"); + None + } + } } }); @@ -1219,7 +2051,56 @@ impl Compositor { } else { None }; - let webcam_draw = webcam_planes.as_ref().map(|(wy, wuv)| { + // Effet d'arriere-plan : le mode vient de la scene, le masque par pixel de + // l'inference. Les DEUX sont requis — un mode sans masque rendrait la + // webcam invisible en detourage, donc tant que rien n'a ete segmente on + // dessine la piste telle quelle. C'est aussi ce qui rend le premier + // lancement gracieux, le temps que l'inference rende son premier masque. + // + // Calcule ICI, avant le draw comme avant l'ombre : les deux en dependent. + let (effect_code, blur_intensity, webcam_bg) = { + let has_mask = self.webcam_mask.borrow().is_some(); + let effect = scene_ref + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .filter(|_| has_mask) + .map(|e| (e.shader_code(), e)) + .filter(|(code, _)| *code > 0.0); + match effect { + // Fond personnalise : on PEINT le fond dans la bulle, puis on y + // decoupe la camera par-dessus — le melange alpha donne + // `lerp(fond, camera, personne)`, soit exactement ce que la branche + // « mode 3 » du shader calculait, mais pour les TROIS sortes de + // fond. Le shader ne sait peindre qu'une couleur plate sous le + // masque ; degrades et images y tombaient sur du noir, et le defaut + // EST une image. + Some((code, e)) if code > 2.5 => { + // Sans piste webcam le fond peindrait un rectangle seul dans le + // cadre : il ne se prepare que si la camera se dessine. + let bg = webcam_planes.is_some().then(|| { + self.webcam_bg_draw( + e.background.as_ref(), + g.w_dst, + g.w_px, + g.w_radius, + &dummy, + ) + }); + (1.0, 0.0, bg) + } + Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0), None), + None => (0.0, 0.0, None), + } + }; + // L'ombre se juge sur le mode DE LA SCENE, pas sur `effect_code` : le fond + // personnalise se compose desormais en detourage (code 1) tout en gardant + // sa bulle, et tester le code compose la lui retirerait. Meme lecture que + // `is_cutout` cote Windows. + let is_cutout = matches!( + scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), + Some(e) if e.shader_code() == 1.0 + ) && self.webcam_mask.borrow().is_some(); + let webcam_draw = webcam_planes.as_ref().map(|(wy, wu, wv)| { // COVER-CROP. `src` etait cable a [0,0,1,1], donc la texture entiere // etait etiree sur la boite quelle que soit sa forme : le facteur de // deformation valait exactement `box_ar / cam_ar`. Invisible en PiP @@ -1255,22 +2136,36 @@ impl Compositor { quad_px: g.w_px, radius_px: g.w_radius, mode: 0.0, + // `color.a` porte l'alpha du decoupage (`color.a * personne`) ; le + // RGB n'est plus lu, le fond ayant deja ete peint sous la camera. color: [0.0, 0.0, 0.0, 1.0], + // `fx.xy` = etendue valide de la texture webcam, par quoi le + // shader divise `uv` pour retomber dans l'espace du masque ; + // `fx.z` = mode, `fx.w` = intensite du flou. Contrat commun aux + // trois back-ends, cf. `layer.wgsl` et `webcam-segmentation.md`. + fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], src_prev: [u0, cv0, u1, cv1], dst_prev: g.w_dst_prev, - mb: [g.mb_taps, 1.0, 1.0, 0.0], + mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], ..Default::default() }; - self.make_bind(&cb, Some((wy, wuv)), &dummy) + // Le masque est lie par `make_bind` sur tous les draws, pas seulement + // celui-ci : le layout l'exige (cf. `tex_entry(4)`). + self.make_bind(&cb, Some((wy, wu, wv)), &dummy) }); // OMBRE de la camera. Pas dans les presets « bloc » (dual-frame, // vertical-stack) : la camera y est collee a l'ecran comme une tuile, // et une ombre entre les deux dessinerait une couture. Meme condition // que macOS. + // + // Pas en detourage non plus : l'ombre appartient a la bulle PiP, et en + // detourage il n'y a plus de bulle — une ombre portee par un rectangle + // devenu invisible se lit comme un artefact. let webcam_shadow = (cfg.shadow && g.shape_fade > 0.0 && webcam_draw.is_some() + && !is_cutout && !matches!( g.scene_preset.as_deref(), Some("dual-frame") | Some("vertical-stack") @@ -1410,7 +2305,7 @@ impl Compositor { // la lit. let (buf, bind) = self.make_bind( &cb, - Some((&self.ann_copy_view, &self.ann_copy_view)), + Some((&self.ann_copy_view, &self.ann_copy_view, &self.ann_copy_view)), &dummy, ); ann_draws.push(AnnDraw::plain(buf, bind)); @@ -1470,7 +2365,7 @@ impl Compositor { fx: [0.0, 0.0, 1.0, 1.0], ..Default::default() }; - let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); ann_draws.push(AnnDraw { _buf: buf, _glyphs: None, @@ -1619,7 +2514,7 @@ impl Compositor { }; // Atlas R8 au binding 1 (texY) que le mode 11 echantillonne. let (buf, bind) = - self.make_bind(&cb, Some((&glyphs.view, &glyphs.view)), &dummy); + self.make_bind(&cb, Some((&glyphs.view, &glyphs.view, &glyphs.view)), &dummy); ann_draws.push(AnnDraw { _buf: buf, _glyphs: Some(glyphs), @@ -1675,21 +2570,13 @@ impl Compositor { .as_deref() .and_then(|t| sprites.get(t)) .or_else(|| sprites.get("arrow"))?; - // Charge (ou recupere du cache) le sprite. Emprunt isole AVANT le - // borrow_mut, comme cote macOS (piege du double emprunt 1re frame). - let cached = self.img_cache.borrow().get(sprite.path.as_str()).cloned(); - let (tex, iw, ih) = match cached { - Some(v) => v, - None => match self.load_image_texture(&sprite.path) { - Ok(v) => { - self.img_cache.borrow_mut().insert(sprite.path.clone(), v.clone()); - v - } - Err(e) => { - eprintln!("[curseur] sprite \"{}\" : {e:#}", sprite.path); - return None; - } - }, + // Charge (ou recupere du cache) le sprite. + let (tex, iw, ih) = match self.cached_image(&sprite.path) { + Ok(v) => v, + Err(e) => { + eprintln!("[curseur] sprite \"{}\" : {e:#}", sprite.path); + return None; + } }; // Ratio preserve : le sprite tient dans un carre de `size_px` de cote. let ar = iw as f32 / ih.max(1) as f32; @@ -1774,7 +2661,7 @@ impl Compositor { } }; // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. - let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); bufs.push(buf); binds.push(bind); } @@ -1881,6 +2768,14 @@ impl Compositor { rpass.set_bind_group(0, bind, &[]); rpass.draw(0..4, 0..1); } + // Fond personnalise : ENTRE l'ombre et la camera. C'est ce sandwich qui + // remplace la branche « mode 3 » du shader — la camera, decoupee, se + // fond dessus par alpha ; l'ombre reste dessous, elle appartient a la + // bulle et non a son contenu. + if let Some(bg) = &webcam_bg { + rpass.set_bind_group(0, &bg.bind, &[]); + rpass.draw(0..4, 0..1); + } if let Some((_buf, bind)) = &webcam_draw { rpass.set_bind_group(0, bind, &[]); rpass.draw(0..4, 0..1); @@ -1961,9 +2856,9 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline_add); - let w = 1.0 / c.binds.len() as f64; - rpass.set_blend_constant(wgpu::Color { r: w, g: w, b: w, a: w }); - for bind in &c.binds { + for (k, bind) in c.binds.iter().enumerate() { + let w = crate::frame_geometry::cursor_tap_weight(k as u32, c.binds.len() as u32) as f64; + rpass.set_blend_constant(wgpu::Color { r: w, g: w, b: w, a: w }); rpass.set_bind_group(0, bind, &[]); rpass.draw(0..4, 0..1); } @@ -2075,7 +2970,387 @@ impl Compositor { Ok(()) } - /// Soumet la copie RT -> staging de la frame COURANTE sans l'attendre, puis + /// Construit (ou reconstruit apres resize) les cibles et pipelines YUV. + fn ensure_yuv(&self) -> Result<()> { + // I420 par defaut : c'est le seul format que l'encodeur software sait + // lire, donc le seul que l'export utilise aujourd'hui. + self.ensure_yuv_fmt(YuvFormat::I420) + } + + /// La disposition du buffer de staging pour un format donne, sans rien + /// construire. Existe pour que le test puisse verifier l'arithmetique sans + /// GPU — c'est elle qui doit correspondre a ce que VAAPI attend, et une + /// erreur d'un octet y donnerait une image decalee plutot qu'une panne. + pub fn yuv_layout_for(w: u32, h: u32, fmt: YuvFormat) -> (u32, u32, u64, u64) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let bpr_y = w.div_ceil(256) * 256; + let chroma_row_bytes = match fmt { + YuvFormat::I420 => cw, + YuvFormat::Nv12 => cw * 2, + }; + let bpr_uv = chroma_row_bytes.div_ceil(256) * 256; + let size_y = u64::from(bpr_y) * u64::from(h); + let size_uv = u64::from(bpr_uv) * u64::from(ch); + let total = match fmt { + YuvFormat::I420 => size_y + 2 * size_uv, + YuvFormat::Nv12 => size_y + size_uv, + }; + (bpr_y, bpr_uv, size_y, total) + } + + /// Comme `ensure_yuv`, pour un format donne. Reconstruit tout si le format + /// change : les cibles, les pipelines et la disposition du buffer en + /// dependent toutes. + fn ensure_yuv_fmt(&self, fmt: YuvFormat) -> Result<()> { + let (w, h) = (self.render_w, self.render_h); + if let Some(t) = self.yuv.borrow().as_ref() { + if t.w == w && t.h == h && t.fmt == fmt { + return Ok(()); + } + } + // 4:2:0 : les plans de chrominance font la moitie, arrondie au superieur + // pour ne jamais perdre la derniere colonne/ligne d'une dimension impaire. + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let gpu = &self.gpu; + + let mk = |label: &str, tw: u32, th: u32, f: wgpu::TextureFormat| { + gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { width: tw, height: th, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: f, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }) + }; + let r8 = wgpu::TextureFormat::R8Unorm; + let y = mk("yuv-y", w, h, r8); + let y_view = y.create_view(&wgpu::TextureViewDescriptor::default()); + + let module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("yuv"), + source: wgpu::ShaderSource::Wgsl(include_str!("vk_shaders/yuv.wgsl").into()), + }); + let bgl = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("yuv-bgl"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + // Sampler LINEAIRE : c'est lui qui fait le sous-echantillonnage 2x2 des + // plans de chrominance. Avec un `Nearest` on prendrait un pixel sur + // quatre au lieu de leur moyenne, ce qui aliase visiblement les bords. + let samp = gpu.device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("yuv-samp"), + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + let bind = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("yuv-bg"), + layout: &bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(&self.rt_view) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&samp) }, + ], + }); + let layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("yuv-pl"), + bind_group_layouts: &[&bgl], + push_constant_ranges: &[], + }); + let mk_pipe = |entry: &str, label: &str, target: wgpu::TextureFormat| { + gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs_fullscreen"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some(entry), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: target, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }) + }; + + let bpr_y = w.div_ceil(256) * 256; + // La LARGEUR EN OCTETS d'une ligne de chrominance, pas en texels : en NV12 + // le plan est `Rg8Unorm`, donc 2 octets par texel. En 1080p, I420 donne + // 960 -> 1024 et NV12 1920 -> 2048. + let chroma_row_bytes = match fmt { + YuvFormat::I420 => cw, + YuvFormat::Nv12 => cw * 2, + }; + let bpr_uv = chroma_row_bytes.div_ceil(256) * 256; + let size_y = u64::from(bpr_y) * u64::from(h); + let size_uv = u64::from(bpr_uv) * u64::from(ch); + let (chroma, off_v, total) = match fmt { + YuvFormat::I420 => { + let u = mk("yuv-u", cw, ch, r8); + let v = mk("yuv-v", cw, ch, r8); + let d = wgpu::TextureViewDescriptor::default(); + let (u_view, v_view) = (u.create_view(&d), v.create_view(&d)); + ( + Chroma::Planar { + _u: u, + _v: v, + u_view, + v_view, + pipe_u: mk_pipe("fs_u", "yuv-u", r8), + pipe_v: mk_pipe("fs_v", "yuv-v", r8), + }, + size_y + size_uv, + size_y + 2 * size_uv, + ) + } + YuvFormat::Nv12 => { + let rg8 = wgpu::TextureFormat::Rg8Unorm; + let uv = mk("yuv-uv", cw, ch, rg8); + let uv_view = uv.create_view(&wgpu::TextureViewDescriptor::default()); + ( + Chroma::Interleaved { + _uv: uv, + uv_view, + pipe_uv: mk_pipe("fs_uv", "yuv-uv", rg8), + }, + // Un seul plan de chrominance : `off_v` duplique `off_u` et + // n'est jamais lu (cf. le commentaire du champ). + size_y, + size_y + size_uv, + ) + } + }; + let targets = YuvTargets { + _y: y, + y_view, + chroma, + fmt, + bind, + pipe_y: mk_pipe("fs_y", "yuv-y", r8), + w, + h, + bpr_y, + bpr_uv, + off_u: size_y, + off_v, + total, + }; + // Les buffers de l'ancienne taille ne conviennent plus. + self.readback_yuv.borrow_mut().free.clear(); + *self.yuv.borrow_mut() = Some(targets); + Ok(()) + } + + /// Pendant YUV de `readback_submit` : convertit le RT en Y/U/V sur le GPU, + /// copie les trois plans dans UN buffer de staging, et recolte la frame + /// precedente. Meme contrat de ring et de profondeur que la version RGBA. + /// + /// Rend les plans avec leur padding : `(w, h, buf)` ou `buf` contient Y a + /// l'offset 0 (stride `align256(w)`), puis U et V (stride `align256(w/2)`). + /// L'appelant recalcule ces strides depuis `w`/`h` — les depadder ici + /// couterait une recopie de plus pour rien, l'encodeur sachant lire un + /// `linesize`. + pub unsafe fn readback_submit_yuv(&self, f: F) -> Result + where + F: FnMut(u32, u32, &[u8]) -> Result<()>, + { + self.ensure_yuv()?; + let (w, h, cw, ch, bpr_y, bpr_uv, off_u, off_v, total) = { + let g = self.yuv.borrow(); + let t = g.as_ref().expect("ensure_yuv"); + (t.w, t.h, t.w.div_ceil(2), t.h.div_ceil(2), t.bpr_y, t.bpr_uv, t.off_u, t.off_v, t.total) + }; + + let buf = { + let mut ring = self.readback_yuv.borrow_mut(); + match ring.free.pop() { + Some(b) => b, + None => self.gpu.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("readback-yuv"), + size: total, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }), + } + }; + + let mut encoder = self + .gpu + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("yuv-convert") }); + { + let g = self.yuv.borrow(); + let t = g.as_ref().expect("ensure_yuv"); + // Une passe par plan : Y toujours, puis U et V separement (I420) ou + // un seul plan entrelace (NV12). + let mut passes: Vec<(&wgpu::TextureView, &wgpu::RenderPipeline)> = + vec![(&t.y_view, &t.pipe_y)]; + match &t.chroma { + Chroma::Planar { u_view, v_view, pipe_u, pipe_v, .. } => { + passes.push((u_view, pipe_u)); + passes.push((v_view, pipe_v)); + } + Chroma::Interleaved { uv_view, pipe_uv, .. } => passes.push((uv_view, pipe_uv)), + } + for (view, pipe) in passes { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("yuv-plane"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + resolve_target: None, + ops: wgpu::Operations { + // Chaque passe reecrit chaque texel : `Load` ferait lire + // une cible dont on va ecraser le contenu. + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(pipe); + pass.set_bind_group(0, &t.bind, &[]); + pass.draw(0..3, 0..1); + } + // `pw` est en TEXELS (`copy_texture_to_buffer` veut une extent), et + // `bpr` en octets : en NV12 le plan de chrominance fait `cw` texels de + // 2 octets, d'ou le meme `cw` avec un `bpr_uv` deux fois plus grand. + let mut copies: Vec<(&wgpu::Texture, u64, u32, u32, u32)> = + vec![(&t._y, 0u64, bpr_y, w, h)]; + match &t.chroma { + Chroma::Planar { _u, _v, .. } => { + copies.push((_u, off_u, bpr_uv, cw, ch)); + copies.push((_v, off_v, bpr_uv, cw, ch)); + } + Chroma::Interleaved { _uv, .. } => copies.push((_uv, off_u, bpr_uv, cw, ch)), + } + for (tex, off, bpr, pw, ph) in copies { + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &buf, + layout: wgpu::TexelCopyBufferLayout { + offset: off, + bytes_per_row: Some(bpr), + rows_per_image: Some(ph), + }, + }, + wgpu::Extent3d { width: pw, height: ph, depth_or_array_layers: 1 }, + ); + } + } + + let idx = self.gpu.context.submit(std::iter::once(encoder.finish())); + let (tx, rx) = std::sync::mpsc::channel(); + buf.slice(..).map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + { + let mut ring = self.readback_yuv.borrow_mut(); + ring.pending.push_back(PendingCopy { buf, idx, rx, w, h, bpr: bpr_y }); + if ring.pending.len() < ring.depth { + return Ok(false); // amorcage, comme la ring RGBA + } + } + self.readback_take_yuv_with(f) + } + + /// Recolte la plus ancienne conversion en vol et la PRESENTE au lecteur sans + /// la copier : `f` recoit la vue mappee telle quelle, lignes paddees a 256 + /// comprises. Rend `false` si la ring est vide. Pendant de `readback_take`. + /// + /// POURQUOI UNE CLOSURE, ET PAS UN `Vec` RENDU. La version precedente faisait + /// `mapped.to_vec()` — 3,3 Mo alloues, copies puis liberes par frame, soit + /// 11,9 Go de va-et-vient sur un export de 3600 frames — dans le seul but que + /// la donnee survive a l'`unmap`. Or l'appelant la recopie immediatement dans + /// l'AVFrame de l'encodeur : la copie intermediaire ne servait que la + /// signature. Avec une closure, le lecteur travaille dans la fenetre ou le + /// buffer est mappe et il n'y a plus qu'une seule copie sur le chemin. + /// + /// LE SLOT EST RENDU MEME SI `f` ECHOUE. Autrement une erreur d'encodage + /// laisserait le buffer mappe et hors de la ring : la frame suivante en + /// allouerait un neuf, et ainsi de suite jusqu'a epuisement de la memoire + /// mappable — un mode de panne bien pire que l'erreur d'origine. + pub unsafe fn readback_take_yuv_with(&self, mut f: F) -> Result + where + F: FnMut(u32, u32, &[u8]) -> Result<()>, + { + let Some(p) = self.readback_yuv.borrow_mut().pending.pop_front() else { + return Ok(false); + }; + self.gpu.device.poll(wgpu::Maintain::WaitForSubmissionIndex(p.idx)); + p.rx + .recv() + .map_err(|_| anyhow::anyhow!("map_async channel (yuv)"))? + .map_err(|e| anyhow::anyhow!("map_async yuv: {e:?}"))?; + // `mapped` et `slice` meurent a la fin du bloc : `unmap` ne peut donc pas + // etre appele pendant qu'une vue est encore accessible (wgpu l'assert). + let r = { + let slice = p.buf.slice(..); + let mapped = slice.get_mapped_range(); + f(p.w, p.h, &mapped) + }; + p.buf.unmap(); + self.readback_yuv.borrow_mut().free.push(p.buf); + r.map(|()| true) + } + + /// Profondeur de la ring YUV. Meme role et memes raisons que + /// `set_readback_depth` pour la ring RGBA. + pub fn set_readback_yuv_depth(&self, depth: usize) -> Result<()> { + let depth = depth.max(1); + // SAFETY : meme contrat que `set_readback_depth` — le drain ne touche que + // des buffers dont la soumission est terminee. + while unsafe { self.readback_take_yuv_with(|_, _, _| Ok(()))? } {} + let mut ring = self.readback_yuv.borrow_mut(); + ring.depth = depth; + while ring.free.len() > depth { + ring.free.pop(); + } + Ok(()) + } + + /// Soumet la copie RT -> staging de la frame COURANTE sans l'attendre, puis /// rend la frame la plus ancienne encore en vol des que la ring est pleine. /// /// PREMIERES FRAMES. Tant que moins de `depth` copies sont en vol, il n'y a @@ -2205,3 +3480,1151 @@ impl Compositor { last.ok_or_else(|| anyhow::anyhow!("readback_direct: aucune frame recoltee")) } } + +// --------------------------------------------------------------------------- +// Tests +// +// Tous rendent de VRAIS pixels sur le device de la machine, et tous sauf un se +// lisent SANS ONNX Runtime : le masque y est pose a la main par +// `set_webcam_mask` et l'inference n'est pas ce qu'ils testent. C'est delibere — +// ce que ce portage ajoute cote GPU doit etre verifiable la ou la bibliotheque +// n'est pas installee, ce qui est le cas de la CI. Meme parti que +// `compositor_macos::tests`, dont ceci est le pendant. +// +// `poc-d3d` etant `cfg(windows)`, le banc `--cfg C8 --scene` qui a prouve le +// chemin Windows n'existe pas ici : ces tests en tiennent lieu, plus le harnais +// visuel opt-in en fin de fichier pour ce qu'une assertion ne peut pas dire. +// --------------------------------------------------------------------------- +#[cfg(test)] +mod tests { + use super::*; + use crate::d3d::Gpu; + use crate::ffi::AVFrame; + + /// NV12 « limited range » (BT.709), les memes valeurs que `yuv709_limited` + /// inverse : 16 rend du noir franc, 235 du blanc franc, 128 une chroma nulle. + const Y_WHITE: u8 = 235; + const Y_BLACK: u8 = 16; + const UV_NEUTRAL: u8 = 128; + + /// `create_auto` et NON `create` : la CI (`rust-linux-compositor-check`, + /// ubuntu-latest) n'a pas de GPU et rend sur lavapipe. Avec la creation + /// hardware-stricte, tous ces tests s'y sauteraient en silence — c'est-a-dire + /// que le seul endroit ou ils tournent automatiquement ne les executerait pas. + fn gpu() -> Option { + match Gpu::create_auto(false) { + Ok(g) => Some(g), + Err(e) => { + eprintln!("pas d'adaptateur Vulkan ({e:#}) — test saute"); + None + } + } + } + + /// Deux `TextureView` NV12-split, comme `linux_frames::nv12_planes` en rend. + /// + /// Les textures ne sont pas retournees : en wgpu une `TextureView` garde la + /// sienne en vie (c'est deja ce dont depend la pyramide de blur du + /// compositeur, qui n'existe que sous forme de vues). + fn nv12_views( + gpu: &Gpu, + w: u32, + h: u32, + luma: impl Fn(u32, u32) -> u8, + ) -> (wgpu::TextureView, wgpu::TextureView, wgpu::TextureView) { + let mut y = vec![0u8; (w * h) as usize]; + for row in 0..h { + for col in 0..w { + y[(row * w + col) as usize] = luma(col, row); + } + } + let (ytex, utex, vtex) = + nv12_textures(gpu, w, h, &y, &vec![UV_NEUTRAL; (w * (h / 2)) as usize]); + let d = wgpu::TextureViewDescriptor::default(); + (ytex.create_view(&d), utex.create_view(&d), vtex.create_view(&d)) + } + + /// Le couple de textures NV12-split (Y `R8Unorm`, UV entrelacee `Rg8Unorm`) + /// exactement comme `linux_frames::CpuFrames::ensure_textures` les alloue. + fn nv12_textures( + gpu: &Gpu, + w: u32, + h: u32, + y: &[u8], + uv: &[u8], + ) -> (wgpu::Texture, wgpu::Texture, wgpu::Texture) { + let mk = |label: &str, format, tw: u32, th: u32| { + gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { width: tw, height: th, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }) + }; + let ytex = mk("test-nv12-y", wgpu::TextureFormat::R8Unorm, w, h); + // Les helpers de test parlent encore NV12 entrelace parce que c'est la + // forme lisible pour ecrire un cas ; le carrier, lui, veut deux plans. + // On desentrelace ici plutot que de reecrire chaque test. + let utex = mk("test-yuv-u", wgpu::TextureFormat::R8Unorm, w / 2, h / 2); + let vtex = mk("test-yuv-v", wgpu::TextureFormat::R8Unorm, w / 2, h / 2); + let u_plane: Vec = uv.iter().step_by(2).copied().collect(); + let v_plane: Vec = uv.iter().skip(1).step_by(2).copied().collect(); + let write = |tex: &wgpu::Texture, data: &[u8], bpr: u32, tw: u32, th: u32| { + gpu.context.write_texture( + wgpu::TexelCopyTextureInfo { + texture: tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bpr), + rows_per_image: Some(th), + }, + wgpu::Extent3d { width: tw, height: th, depth_or_array_layers: 1 }, + ); + }; + write(&ytex, y, w, w, h); + write(&utex, &u_plane, w / 2, w / 2, h / 2); + write(&vtex, &v_plane, w / 2, w / 2, h / 2); + (ytex, utex, vtex) + } + + /// Masque 0 sur la moitie gauche, 255 sur la droite. La frontiere tombe pile + /// au milieu, donc un echantillon pris au quart et un aux trois quarts sont + /// loin du degrade que le filtrage lineaire pose sur la couture. + fn half_mask(w: u32, h: u32) -> Vec { + (0..w * h).map(|i| if i % w < w / 2 { 0u8 } else { 255u8 }).collect() + } + + /// Le buffer de staging exportable doit etre une VRAIE zone partagee : ce que + /// wgpu y ecrit, notre propre mapping doit le relire a l'identique. + /// + /// C'est le seul point reellement incertain de l'export dmabuf, et il se + /// verifie sans encodeur. Si ce test passe, la memoire qu'on remettra a VAAPI + /// est bien celle que le compositeur remplit ; s'il echoue, tout ce qui est + /// bati dessus produirait une image fausse plutot qu'une panne. + #[test] + fn exportable_staging_round_trips_through_wgpu() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + const N: u64 = 4096; + let Some(st) = comp.create_exportable_staging(N) else { + eprintln!("pas d'extensions de memoire externe — test saute"); + return; + }; + assert!(st.fd >= 0, "descripteur dmabuf invalide"); + assert_eq!(st.size, N); + + // Un motif non trivial : un remplissage constant passerait meme si les + // deux cotes regardaient deux zones differentes mais nulles. + let pattern: Vec = (0..N as usize).map(|i| (i * 31 + 7) as u8).collect(); + gpu.context.write_buffer(st.buffer(), 0, &pattern); + gpu.context.submit(std::iter::empty()); + gpu.device.poll(wgpu::Maintain::Wait); + + let got = st.read_back().expect("read_back"); + assert_eq!(got.len(), N as usize); + assert_eq!(got, pattern, "la memoire exportee ne porte pas ce que wgpu y a ecrit"); + } + + /// La disposition NV12 doit etre EXACTEMENT celle que le pilote produit pour + /// une image NV12 lineaire, parce que c'est elle qu'on decrira a VAAPI dans + /// un `AVDRMFrameDescriptor`. Les valeurs ci-dessous ne sont pas devinees : + /// elles ont ete relevees sur ce materiel via `vkGetImageSubresourceLayout` + /// d'une `VkImage` NV12 en `DRM_FORMAT_MOD_LINEAR` (Y pitch 2048, UV a + /// l'offset 2211840, pitch 2048, total 3317760). Un ecart d'un octet ici + /// donnerait une image decalee et non une panne, d'ou le test. + #[test] + fn nv12_layout_matches_what_the_driver_produces() { + let (bpr_y, bpr_uv, off_uv, total) = + Compositor::yuv_layout_for(1920, 1080, YuvFormat::Nv12); + assert_eq!(bpr_y, 2048, "pitch du plan Y"); + assert_eq!(bpr_uv, 2048, "pitch du plan UV entrelace (960 texels x 2 octets)"); + assert_eq!(off_uv, 2_211_840, "offset du plan UV"); + assert_eq!(total, 3_317_760, "taille totale"); + } + + /// I420 reste ce qu'il etait : c'est le format que l'encodeur software lit, + /// et ce test est ce qui garantit qu'ajouter NV12 ne l'a pas deplace. + #[test] + fn i420_layout_is_unchanged() { + let (bpr_y, bpr_uv, off_u, total) = + Compositor::yuv_layout_for(1920, 1080, YuvFormat::I420); + assert_eq!((bpr_y, bpr_uv), (2048, 1024)); + assert_eq!(off_u, 2_211_840); + assert_eq!(total, 2_211_840 + 2 * 1024 * 540); + } + + /// Dessine UN calque plein cadre sur le RT, par-dessus `clear`, et rend le + /// RGBA relu. + /// + /// Court-circuite `compose_frame` a dessein : ces tests-ci isolent le shader + /// et la liaison du masque, pas la geometrie que `plan_frame` decide. + fn draw_one_layer( + comp: &Compositor, + clear: wgpu::Color, + cb: &LayerCB, + planes: (&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView), + ) -> (u32, u32, Vec) { + let dummy = comp.dummy_view(); + let (_buf, bind) = comp.make_bind(cb, Some(planes), &dummy); + let mut encoder = comp.gpu.device.create_command_encoder( + &wgpu::CommandEncoderDescriptor { label: Some("test-layer") }, + ); + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("test-layer-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &comp.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(clear), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&comp.pipeline); + rpass.set_bind_group(0, &bind, &[]); + rpass.draw(0..4, 0..1); + } + comp.gpu.context.submit(std::iter::once(encoder.finish())); + unsafe { comp.readback_direct().expect("readback_direct") } + } + + // ----------------------------------------------------------------------- + // La capture + // ----------------------------------------------------------------------- + + #[test] + fn the_webcam_capture_comes_back_as_interleaved_rgb_at_model_resolution() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + // Moitie gauche noire, moitie droite blanche : la capture doit rendre les + // deux dans le bon sens. Une inversion d'axe passerait un test de taille + // sans se voir. + let (y, u, v) = nv12_views(&gpu, 64, 64, |col, _| if col < 32 { Y_BLACK } else { Y_WHITE }); + + let mut out = Vec::new(); + unsafe { + comp.capture_webcam_rgb( + &y, + &u, + &v, + [0.0, 0.0, 1.0, 1.0], + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + &mut out, + ) + .expect("capture_webcam_rgb"); + } + + let (w, h) = ( + crate::segmentation::MODEL_WIDTH as usize, + crate::segmentation::MODEL_HEIGHT as usize, + ); + assert_eq!(out.len(), w * h * 3, "le modele veut du RGB8 entrelace, sans alpha"); + + let px = |buf: &[u8], col: usize, row: usize| -> [u8; 3] { + let i = (row * w + col) * 3; + [buf[i], buf[i + 1], buf[i + 2]] + }; + let left = px(&out, w / 4, h / 2); + let right = px(&out, 3 * w / 4, h / 2); + assert!(left.iter().all(|&c| c < 24), "moitie gauche pas noire : {left:?}"); + assert!(right.iter().all(|&c| c > 231), "moitie droite pas blanche : {right:?}"); + + // Deuxieme capture sur le meme buffer : c'est le regime etabli (30 fois + // par seconde), et il ne doit ni reallouer ni trainer les octets du tour + // precedent. + let capacity = out.capacity(); + unsafe { + comp.capture_webcam_rgb( + &y, + &u, + &v, + [0.0, 0.0, 1.0, 1.0], + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + &mut out, + ) + .expect("deuxieme capture"); + } + assert_eq!(out.len(), w * h * 3); + assert_eq!(out.capacity(), capacity, "le scratch se realloue d'une frame a l'autre"); + assert_eq!(px(&out, w / 4, h / 2), left); + assert_eq!(px(&out, 3 * w / 4, h / 2), right); + } + + /// Le piege PROPRE a ce backend : `copy_texture_to_buffer` exige un + /// `bytes_per_row` multiple de 256, et le depadder est a la charge de + /// l'appelant. A la resolution livree (256 px, 1024 octets) le padding est nul + /// — donc la resolution livree n'exerce JAMAIS ce chemin. Il faut une largeur + /// qui le fasse : 100 px = 400 octets utiles dans un pas de 512. + /// + /// Un depad rate ne rend pas du bruit, il rend un CISAILLEMENT : chaque ligne + /// glisse de 28 px sur la precedente. D'ou l'echantillonnage sur plusieurs + /// lignes plutot que sur une seule. + #[test] + fn a_capture_whose_rows_need_padding_is_depadded_correctly() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let (y, u, v) = nv12_views(&gpu, 64, 64, |col, _| if col < 32 { Y_BLACK } else { Y_WHITE }); + + let (w, h) = (100usize, 56usize); + assert_ne!((w * 4) % 256, 0, "cette largeur doit justement ETRE mal alignee"); + let mut out = Vec::new(); + unsafe { + comp.capture_webcam_rgb(&y, &u, &v, [0.0, 0.0, 1.0, 1.0], w as u32, h as u32, &mut out) + .expect("capture_webcam_rgb"); + } + assert_eq!(out.len(), w * h * 3, "le padding d'alignement a fuit dans la sortie"); + + let px = |col: usize, row: usize| -> [u8; 3] { + let i = (row * w + col) * 3; + [out[i], out[i + 1], out[i + 2]] + }; + for row in [0usize, h / 3, h / 2, h - 1] { + let left = px(w / 4, row); + let right = px(3 * w / 4, row); + assert!(left.iter().all(|&c| c < 24), "ligne {row}, gauche pas noire : {left:?}"); + assert!(right.iter().all(|&c| c > 231), "ligne {row}, droite pas blanche : {right:?}"); + } + } + + #[test] + fn a_capture_of_zero_size_is_refused_rather_than_rendered() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let (y, u, v) = nv12_views(&gpu, 16, 16, |_, _| Y_WHITE); + let mut out = Vec::new(); + let r = unsafe { comp.capture_webcam_rgb(&y, &u, &v, [0.0, 0.0, 1.0, 1.0], 0, 144, &mut out) }; + assert!(r.is_err(), "une cible de largeur nulle doit etre refusee"); + } + + // ----------------------------------------------------------------------- + // Le masque + // ----------------------------------------------------------------------- + + #[test] + fn the_mask_texture_is_allocated_once_and_a_short_buffer_is_refused() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let (w, h) = (crate::segmentation::MODEL_WIDTH, crate::segmentation::MODEL_HEIGHT); + let mask = vec![255u8; (w * h) as usize]; + + comp.set_webcam_mask(&mask, w, h).expect("premier televersement"); + let first = comp.webcam_mask.borrow().as_ref().map(|m| m.tex.clone()); + comp.set_webcam_mask(&mask, w, h).expect("deuxieme televersement"); + let second = comp.webcam_mask.borrow().as_ref().map(|m| m.tex.clone()); + assert_eq!( + first, second, + "la texture est recreee a chaque frame alors que la resolution du modele est fixe" + ); + + // Un masque trop court doit etre refuse, pas lu hors bornes. + assert!(comp.set_webcam_mask(&mask[..(w * h) as usize - 1], w, h).is_err()); + assert!(comp.set_webcam_mask(&mask, 0, h).is_err()); + comp.clear_webcam_mask(); + assert!(comp.webcam_mask.borrow().is_none()); + } + + /// Le test qui compte : le masque DECOUPE vraiment la camera. + /// + /// Il rend le calque webcam plein cadre avec `fx.z = 1` (detourage) et un + /// masque mi-fond mi-sujet, puis relit les pixels. Il couvre d'un coup les + /// trois choses que le portage ajoute et qu'aucune compilation ne verifie : + /// le televersement R8, la liaison de la texture au binding 4, et la branche + /// `fx.z` de `fs_main` sur un vrai device. + #[test] + fn the_mask_actually_cuts_the_camera_out() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 64, 64).expect("Compositor::new_sized"); + comp.set_webcam_mask(&half_mask(8, 8), 8, 8).expect("set_webcam_mask"); + let (y, u, v) = nv12_views(&gpu, 16, 16, |_, _| Y_WHITE); + + // Fond bleu franc : une couleur que la camera (blanche, chroma neutre) ne + // peut pas produire, donc « il reste du bleu » signifie « la camera a ete + // decoupee ici ». + let (rw, _, rgba) = draw_one_layer( + &comp, + wgpu::Color { r: 0.0, g: 0.0, b: 1.0, a: 1.0 }, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src: [0.0, 0.0, 1.0, 1.0], + quad_px: [64.0, 64.0], + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + // fx.xy = etendue valide (toute la texture ici), fx.z = 1 -> detourage. + fx: [1.0, 1.0, 1.0, 0.0], + src_prev: [0.0, 0.0, 1.0, 1.0], + dst_prev: [0.0, 0.0, 1.0, 1.0], + mb: [1.0, 1.0, 1.0, 0.0], + ..Default::default() + }, + (&y, &u, &v), + ); + + let px = |col: usize, row: usize| -> [u8; 4] { + let i = (row * rw as usize + col) * 4; + [rgba[i], rgba[i + 1], rgba[i + 2], rgba[i + 3]] + }; + assert_eq!(px(16, 32), [0, 0, 255, 255], "masque a 0 : le fond doit rester visible"); + assert_eq!(px(48, 32), [255, 255, 255, 255], "masque a 255 : la camera doit rester opaque"); + } + + /// Meme montage, mode fond personnalise (`fx.z = 3`) : la ou le masque dit + /// « fond », le shader doit peindre `color` — c'est le seul mode ou + /// `LayerCB::color` cesse d'etre du noir opaque decoratif et porte une valeur + /// que le portage doit transmettre. + #[test] + fn the_custom_background_colour_replaces_the_masked_out_pixels() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 64, 64).expect("Compositor::new_sized"); + comp.set_webcam_mask(&half_mask(8, 8), 8, 8).expect("set_webcam_mask"); + let (y, u, v) = nv12_views(&gpu, 16, 16, |_, _| Y_WHITE); + + let (rw, _, rgba) = draw_one_layer( + &comp, + wgpu::Color::BLACK, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src: [0.0, 0.0, 1.0, 1.0], + quad_px: [64.0, 64.0], + mode: 0.0, + color: [1.0, 0.0, 0.0, 1.0], + fx: [1.0, 1.0, 3.0, 0.0], + src_prev: [0.0, 0.0, 1.0, 1.0], + dst_prev: [0.0, 0.0, 1.0, 1.0], + mb: [1.0, 1.0, 1.0, 0.0], + ..Default::default() + }, + (&y, &u, &v), + ); + let px = |col: usize, row: usize| -> [u8; 4] { + let i = (row * rw as usize + col) * 4; + [rgba[i], rgba[i + 1], rgba[i + 2], rgba[i + 3]] + }; + assert_eq!(px(16, 32), [255, 0, 0, 255], "fond masque : la couleur custom doit peindre"); + assert_eq!(px(48, 32), [255, 255, 255, 255], "sujet : la camera doit rester intacte"); + } + + // ----------------------------------------------------------------------- + // `compose_frame` de bout en bout + // + // Les tests ci-dessus prouvent les pieces ; ceux-ci prouvent le CABLAGE — que + // `compose_frame` porte bien `fx`/`color` sur le calque webcam, qu'il lie le + // masque, et qu'il ne leve `fx.z` qu'une fois un masque reellement televerse. + // Ils passent par de vraies `AVFrame` porteuses d'un carrier `VkFrameTex`, + // donc par le MEME `nv12_srvs` que le decodeur : aucun raccourci n'est pris + // sur le seam de frame. + // ----------------------------------------------------------------------- + + /// Une `AVFrame` du backend Linux. `compose_frame` n'en lit que `format`, + /// `data[0]`, `width` et `height` : le reste peut rester a zero. + struct FakeFrame { + frame: Box, + } + + impl FakeFrame { + fn new(gpu: &Gpu, w: u32, h: u32, luma: impl Fn(u32, u32) -> u8) -> FakeFrame { + let mut y = vec![0u8; (w * h) as usize]; + for row in 0..h { + for col in 0..w { + y[(row * w + col) as usize] = luma(col, row); + } + } + FakeFrame::from_planes(gpu, w, h, &y, &vec![UV_NEUTRAL; (w * (h / 2)) as usize]) + } + + fn from_planes(gpu: &Gpu, w: u32, h: u32, y: &[u8], uv: &[u8]) -> FakeFrame { + let (ytex, utex, vtex) = nv12_textures(gpu, w, h, y, uv); + // Le carrier que `linux_frames::nv12_planes` et `carrier_dims` + // deballent. `Box::into_raw` ici, `Box::from_raw` dans `Drop` — c'est + // exactement la mecanique de `CpuFrames::attach_carrier`. + let carrier = Box::into_raw(Box::new(crate::linux_frames::VkFrameTex { + y: ytex, + u: utex, + v: vtex, + width: w, + height: h, + })) as *mut u8; + let mut frame: Box = Box::new(unsafe { std::mem::zeroed() }); + // Le sentinel « buffer GPU natif dans data[0] », le meme que pose + // `CpuFrames::present`. + frame.format = crate::ffi::AVPixelFormat::AV_PIX_FMT_D3D11 as i32; + frame.data[0] = carrier; + frame.width = w as i32; + frame.height = h as i32; + FakeFrame { frame } + } + + fn as_ptr(&self) -> *const AVFrame { + &*self.frame as *const AVFrame + } + } + + impl Drop for FakeFrame { + fn drop(&mut self) { + if !self.frame.data[0].is_null() { + unsafe { + drop(Box::from_raw( + self.frame.data[0] as *mut crate::linux_frames::VkFrameTex, + )); + } + self.frame.data[0] = std::ptr::null_mut(); + } + } + } + + /// Scene PiP minimale. `effect` est le JSON de `webcamEffect` (`"null"` pour + /// aucun). + /// + /// `effects.shadow` vaut 0 A DESSEIN : ce curseur ne pilote plus que l'ombre + /// de l'ecran, alors que celle du PiP est fixe (`WEBCAM_SHADOW_OPACITY`) et ne + /// depend que de `cfg.shadow`. Le mettre a zero est donc ce qui isole les + /// deux — sinon un test sur `cfg.shadow` mesure les deux ombres a la fois et + /// ne dit plus rien de la camera. + fn pip_scene_json(effect: &str) -> String { + format!( + r##"{{"clips":[], + "layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle", + "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects":{{"padding":0.18,"blur":false,"shadow":0,"roundnessFrac":0.05,"motionBlur":0}}, + "background":{{"kind":"color","color":"#0080ff"}}, + "zoomRegions":[],"annotations":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0, + "clipToBounds":false,"theme":"default"}}, + "cropByClip":[], + "webcamEffect":{effect}, + "output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + } + + /// Compose une frame et rend le RGBA du RT. `screen` est gris moyen, `webcam` + /// blanche : le blanc franc devient alors la SIGNATURE de la camera, une + /// couleur qu'aucun autre calque de cette scene ne produit, donc comptable + /// sans connaitre la geometrie du PiP. + /// + /// Le fond est un bleu franc et NON du noir : le PiP par defaut tombe dans la + /// marge, hors de l'ecran, et une ombre noire sur un fond noir ne se voit + /// pas — le controle du test d'ombre passerait alors pour une suppression + /// reussie. + /// + /// `set_live_params(live_params_from_scene(..))` n'est PAS decoratif : padding, + /// effets et forme de la webcam transitent par `LiveParams` et non par la + /// scene brute. L'omettre laisse la scene parser correctement puis etre + /// ignoree, et le rendu tombe sur les defauts. + fn compose_pip( + comp: &Compositor, + gpu: &Gpu, + effect: &str, + shadow: bool, + ) -> Vec { + let scene = Scene::from_json(&pip_scene_json(effect)).expect("scene json"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_has_webcam(true); + comp.set_scene(Some(scene)); + + let screen = FakeFrame::new(gpu, 128, 128, |_, _| 126); + let webcam = FakeFrame::new(gpu, 64, 64, |_, _| Y_WHITE); + let mut cfg = Cfg::c8(); + cfg.bg_blur = false; + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.mblur_n = 1; + cfg.shadow = shadow; + unsafe { + comp.compose_frame(screen.as_ptr(), webcam.as_ptr(), 0.0, &cfg) + .expect("compose_frame"); + let (_, _, rgba) = comp.readback_direct().expect("readback_direct"); + rgba + } + } + + /// Pixels quasi blancs = pixels de camera encore visibles. + fn camera_pixels(rgba: &[u8]) -> usize { + rgba.chunks_exact(4) + .filter(|px| px[0] > 240 && px[1] > 240 && px[2] > 240) + .count() + } + + const NO_EFFECT: &str = "null"; + const CUTOUT: &str = + r#"{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":null}"#; + + /// Le piege que le brief nomme : un mode SANS masque ne doit rien changer. + /// + /// `effect_code` doit rester a 0 tant que rien n'a ete segmente, sinon le + /// detourage rend une webcam invisible sur les premieres frames — le temps que + /// l'inference rende son premier masque, c'est-a-dire a chaque ouverture de + /// l'editeur. L'assertion est octet pour octet : « inchange » ne souffre pas + /// d'a-peu-pres. + #[test] + fn a_mode_without_a_mask_composites_exactly_like_no_effect_at_all() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let plain = compose_pip(&comp, &gpu, NO_EFFECT, true); + let requested = compose_pip(&comp, &gpu, CUTOUT, true); + assert!( + comp.webcam_mask.borrow().is_none(), + "aucun masque n'a ete televerse : `modelPath` est absent, donc rien ne segmente" + ); + assert!( + camera_pixels(&plain) > 200, + "la camera n'est pas a l'ecran, le test ne prouve rien" + ); + assert_eq!(plain, requested, "un mode sans masque a change des pixels"); + } + + /// Et une fois le masque la, le detourage doit VRAIMENT decouper — dans la + /// bonne proportion. Le masque couvre la moitie de la camera, donc la moitie + /// de ses pixels doit disparaitre. Compter plutot que d'echantillonner un + /// point evite de coder en dur la geometrie du PiP, qui appartient a + /// `plan_frame` et non a ce portage. + #[test] + fn compose_frame_cuts_the_camera_out_once_a_mask_exists() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let whole = camera_pixels(&compose_pip(&comp, &gpu, NO_EFFECT, true)); + assert!(whole > 200, "la camera n'est pas a l'ecran, le test ne prouve rien"); + + let (mw, mh) = (crate::segmentation::MODEL_WIDTH, crate::segmentation::MODEL_HEIGHT); + comp.set_webcam_mask(&half_mask(mw, mh), mw, mh).expect("set_webcam_mask"); + let cut = camera_pixels(&compose_pip(&comp, &gpu, CUTOUT, true)); + + let expected = whole as f32 / 2.0; + assert!( + (cut as f32 - expected).abs() < expected * 0.15, + "detourage : {cut} pixels de camera restants pour ~{expected:.0} attendus \ + (entier : {whole})" + ); + } + + /// L'ombre portee du PiP doit disparaitre en detourage : une ombre projetee + /// par un rectangle devenu invisible se lit comme un artefact. Le test le + /// prouve sans jamais localiser l'ombre — en detourage, `cfg.shadow` ne doit + /// plus rien changer du tout. + /// + /// Le controle est ce qui empeche l'assertion d'etre vide : sans effet, + /// `cfg.shadow` DOIT changer des pixels, sinon la premiere moitie passerait + /// aussi pour une scene ou aucune ombre n'a jamais ete dessinee. + #[test] + fn the_pip_shadow_is_suppressed_in_cutout_mode() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + assert_ne!( + compose_pip(&comp, &gpu, NO_EFFECT, true), + compose_pip(&comp, &gpu, NO_EFFECT, false), + "controle : sans effet, l'ombre du PiP doit bel et bien se voir" + ); + + let (mw, mh) = (crate::segmentation::MODEL_WIDTH, crate::segmentation::MODEL_HEIGHT); + comp.set_webcam_mask(&half_mask(mw, mh), mw, mh).expect("set_webcam_mask"); + assert_eq!( + compose_pip(&comp, &gpu, CUTOUT, true), + compose_pip(&comp, &gpu, CUTOUT, false), + "en detourage, l'ombre est encore dessinee" + ); + } + + /// Le tour complet, celui qui a besoin d'ONNX Runtime : capture -> inference + /// -> masque -> composite, entraine par `compose_frame` seul. Se saute + /// proprement sans la bibliotheque, ce que fait la CI — cf. + /// `segmentation::runtime_available`. + #[test] + fn the_whole_loop_produces_a_mask_from_compose_frame_alone() { + if !crate::segmentation::runtime_available() { + eprintln!("ONNX Runtime absent (ORT_DYLIB_PATH) — test saute"); + return; + } + let model = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx"); + if !model.is_file() { + eprintln!("modele absent ({}) — test saute", model.display()); + return; + } + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let effect = format!( + r#"{{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":{}}}"#, + serde_json::to_string(&model.to_string_lossy()).expect("chemin serialisable") + ); + + // Le limiteur est a 30 Hz : une frame par tour ne suffirait pas, et + // l'inference est asynchrone. On laisse au worker le temps de rendre un + // masque, sans jamais l'attendre dans le rendu — ce qui est precisement le + // contrat. + let mut uploaded = false; + for _ in 0..40 { + let _ = compose_pip(&comp, &gpu, &effect, true); + if comp.webcam_mask.borrow().is_some() { + uploaded = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(40)); + } + assert!( + uploaded, + "aucun masque n'est remonte : la boucle capture -> inference -> upload est rompue" + ); + assert!(!*comp.seg_failed.borrow(), "la segmentation s'est eteinte d'elle-meme"); + } + + // ----------------------------------------------------------------------- + // Harnais visuel (opt-in) + // + // Les tests ci-dessus prouvent le mecanisme sur des images synthetiques, ou le + // masque est pose a la main et donc trivialement juste. Ils ne peuvent rien + // dire de la QUALITE du masque que le modele produit sur une vraie camera — et + // « un masque qui composite » n'est pas la meme affirmation que « un masque qui + // est correct ». + // + // Meme forme d'opt-in que `tests/compose_linux.rs` (variable d'environnement + + // skip propre), et pour la meme raison : ca rend sur GPU et ca lit un fichier + // que le depot ne porte pas. + // + // ``` + // ORT_DYLIB_PATH=/chemin/libonnxruntime.so \ + // OPENSCREEN_SEG_CAM=camera.png \ + // OPENSCREEN_SEG_VISUAL=target/seg \ + // cargo test -p openscreen-compositor --lib seg_visual -- --nocapture + // ``` + // ----------------------------------------------------------------------- + + /// RGB8 -> NV12 BT.709 limited. Inverse EXACT de `yuv709_limited` dans + /// `layer.wgsl` : une autre matrice ferait deriver les couleurs du rendu et on + /// croirait a un bug du compositeur la ou il n'y aurait qu'une conversion + /// d'entree fausse. + fn rgb_to_nv12(rgb: &[u8], w: u32, h: u32) -> (Vec, Vec) { + let luma = |i: usize| -> (f32, f32, f32, f32) { + let (r, g, b) = ( + rgb[i * 3] as f32 / 255.0, + rgb[i * 3 + 1] as f32 / 255.0, + rgb[i * 3 + 2] as f32 / 255.0, + ); + (r, g, b, 0.2126 * r + 0.7152 * g + 0.0722 * b) + }; + let mut y = vec![0u8; (w * h) as usize]; + for i in 0..(w * h) as usize { + let (_, _, _, yl) = luma(i); + y[i] = (16.0 + 219.0 * yl).round().clamp(0.0, 255.0) as u8; + } + // Chroma au plus proche voisin : l'echantillon en haut a gauche de chaque + // bloc 2x2. Un vrai filtre ne changerait rien a ce que ce harnais donne a + // voir. + let mut uv = vec![0u8; (w * (h / 2)) as usize]; + for row in 0..h / 2 { + for col in 0..w / 2 { + let (r, _, b, yl) = luma(((row * 2) * w + col * 2) as usize); + let cb = 128.0 + 224.0 * ((b - yl) / 1.8556); + let cr = 128.0 + 224.0 * ((r - yl) / 1.5748); + let o = (row * w + col * 2) as usize; + uv[o] = cb.round().clamp(0.0, 255.0) as u8; + uv[o + 1] = cr.round().clamp(0.0, 255.0) as u8; + } + } + (y, uv) + } + + fn frame_from_png(gpu: &Gpu, path: &std::path::Path) -> FakeFrame { + let img = image::open(path) + .unwrap_or_else(|e| panic!("{} : {e}", path.display())) + .to_rgb8(); + // NV12 veut des dimensions paires ; on rogne d'un pixel plutot que de + // reechantillonner. + let (w, h) = (img.width() & !1, img.height() & !1); + let src = img.as_raw(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for row in 0..h { + let (d, s) = ((row * w * 3) as usize, (row * img.width() * 3) as usize); + rgb[d..d + (w * 3) as usize].copy_from_slice(&src[s..s + (w * 3) as usize]); + } + let (y, uv) = rgb_to_nv12(&rgb, w, h); + FakeFrame::from_planes(gpu, w, h, &y, &uv) + } + + #[test] + fn seg_visual_renders_the_four_modes_from_a_real_photo() { + let (Ok(out_dir), Ok(cam)) = ( + std::env::var("OPENSCREEN_SEG_VISUAL"), + std::env::var("OPENSCREEN_SEG_CAM"), + ) else { + eprintln!( + "harnais visuel : OPENSCREEN_SEG_VISUAL + OPENSCREEN_SEG_CAM absents — saute" + ); + return; + }; + if !crate::segmentation::runtime_available() { + eprintln!("ONNX Runtime absent (ORT_DYLIB_PATH) — saute"); + return; + } + let model = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx"); + let Some(gpu) = gpu() else { return }; + std::fs::create_dir_all(&out_dir).expect("dossier de sortie"); + + let (rw, rh) = (1280u32, 720u32); + let comp = Compositor::new_sized(&gpu, rw, rh).expect("Compositor::new_sized"); + let webcam = frame_from_png(&gpu, std::path::Path::new(&cam)); + let screen = match std::env::var("OPENSCREEN_SEG_SCREEN") { + Ok(p) => frame_from_png(&gpu, std::path::Path::new(&p)), + // Sans capture d'ecran sous la main, un damier : il rend le detourage + // lisible, la ou un aplat laisserait croire a un fond simplement peint. + Err(_) => FakeFrame::new(&gpu, 640, 360, |col, row| { + if (col / 40 + row / 40) % 2 == 0 { 180 } else { 60 } + }), + }; + let model_json = serde_json::to_string(&model.to_string_lossy()).expect("chemin"); + + let mut wrote = Vec::new(); + for (name, effect) in [ + ("00-none", "null".to_string()), + ("01-cutout", format!(r#"{{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":{model_json}}}"#)), + ("02-blur", format!(r#"{{"mode":"blur","blurIntensity":0.8,"background":null,"modelPath":{model_json}}}"#)), + ("03-custom", format!(r##"{{"mode":"custom","blurIntensity":0,"background":{{"kind":"color","color":"#ff2d95"}},"modelPath":{model_json}}}"##)), + ] { + // Le masque arrive de facon asynchrone : on tourne jusqu'a ce qu'il + // soit la, ce qui est aussi une verification en soi — la boucle du + // rendu ne l'attend jamais. + let mut rgba = Vec::new(); + for _ in 0..60 { + rgba = compose_visual(&comp, &screen, &webcam, &effect); + if effect == "null" || comp.webcam_mask.borrow().is_some() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(30)); + } + let path = format!("{out_dir}/seg-{name}.png"); + image::RgbaImage::from_raw(rw, rh, rgba) + .expect("dimensions du readback") + .save(&path) + .unwrap_or_else(|e| panic!("ecriture {path} : {e}")); + wrote.push(path); + } + for p in &wrote { + println!("wrote {p}"); + } + assert!( + comp.webcam_mask.borrow().is_some(), + "aucun masque n'a ete produit : les trois modes d'effet sont sans objet" + ); + } + + /// Camera grand format (rect force via `webcamRect`), pour que le masque + /// occupe une bonne part de l'image et se juge a taille reelle. + fn compose_visual( + comp: &Compositor, + screen: &FakeFrame, + webcam: &FakeFrame, + effect: &str, + ) -> Vec { + let json = format!( + r##"{{"clips":[], + "layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle", + "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, + "webcamRect":{{"x":0.06,"y":0.10,"width":0.55,"height":0.72}}}}, + "effects":{{"padding":0.10,"blur":false,"shadow":1,"roundnessFrac":0.02,"motionBlur":0}}, + "background":{{"kind":"gradient","angleDeg":45,"stops":["#1b2a4a","#0b0f1a"]}}, + "zoomRegions":[],"annotations":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0, + "clipToBounds":false,"theme":"default"}}, + "cropByClip":[], + "webcamEffect":{effect}, + "output":{{"width":1280,"height":720,"fps":30}}}}"## + ); + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_has_webcam(true); + comp.set_scene(Some(scene)); + let mut cfg = Cfg::c8(); + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.mblur_n = 1; + unsafe { + comp.compose_frame(screen.as_ptr(), webcam.as_ptr(), 0.0, &cfg) + .expect("compose_frame"); + let (_, _, rgba) = comp.readback_direct().expect("readback_direct"); + rgba + } + } +} + +// --------------------------------------------------------------------------- +// Staging exportable en dmabuf +// --------------------------------------------------------------------------- + +/// Un buffer de staging dont la MEMOIRE est exportable en dmabuf, pour qu'un +/// encodeur materiel puisse la lire sans repasser par le CPU. +/// +/// POURQUOI IL EN FAUT UN DEUXIEME, ET PAS UN DRAPEAU SUR L'EXISTANT. wgpu +/// n'expose aucun moyen de demander une allocation exportable : il faut la +/// fabriquer soi-meme et la lui confier. Or `buffer_from_raw` construit un +/// `Buffer { block: None }` -- wgpu accepte d'y ECRIRE (c'est une cible de +/// `copy_texture_to_buffer` comme une autre) mais ne peut pas le faire lire par +/// le CPU, sa mecanique de mapping passant par ce bloc qu'il ne possede pas. +/// Le chemin logiciel, lui, DOIT le lire. Les deux ne peuvent donc pas partager +/// un buffer, et l'export choisit lequel il alloue selon l'encodeur retenu. +/// +/// La memoire est demandee HOST_VISIBLE et HOST_COHERENT pour que la +/// verification puisse la relire directement et sans invalidation ; un chemin +/// purement GPU pourrait se passer des deux. +pub struct ExportableStaging { + /// Vue wgpu, utilisable comme destination de copie. En `Option` UNIQUEMENT + /// pour pouvoir la relacher explicitement avant la memoire dans `Drop`, cf. + /// l'ordre impose la-bas. + buffer: Option, + /// Le descripteur a passer au consommateur. Possede : ferme dans `Drop`. + pub fd: i32, + pub size: u64, + device: ash::Device, + memory: ash::vk::DeviceMemory, +} + +impl ExportableStaging { + /// La cible de copie a passer a wgpu. + pub fn buffer(&self) -> &wgpu::Buffer { + self.buffer.as_ref().expect("buffer relache") + } +} + +impl ExportableStaging { + /// Relit la memoire exportee telle que le GPU l'a laissee. + /// + /// Passe par `vkMapMemory` et NON par wgpu, pour la raison ci-dessus. C'est + /// ce qui permet de verifier le contenu sans encodeur : si ces octets sont + /// ceux du chemin de relecture normal, la memoire exportee porte bien + /// l'image composee. + pub fn read_back(&self) -> Result> { + unsafe { + let p = self + .device + .map_memory(self.memory, 0, self.size, ash::vk::MemoryMapFlags::empty()) + .map_err(|e| anyhow::anyhow!("vkMapMemory: {e}"))?; + let out = std::slice::from_raw_parts(p as *const u8, self.size as usize).to_vec(); + self.device.unmap_memory(self.memory); + Ok(out) + } + } +} + +impl Drop for ExportableStaging { + fn drop(&mut self) { + // L'ORDRE EST LE FOND DU SUJET, et le premier jet le faisait a l'envers : + // il detruisait le `VkBuffer` puis liberait la memoire, alors que wgpu + // detruit DEJA le buffer quand son wrapper tombe -- double liberation, + // et par-dessus, memoire liberee alors qu'un buffer y etait encore lie. + // + // Le partage est donc : wgpu possede le HANDLE (il l'a recu par + // `buffer_from_raw` et le detruira), nous possedons la MEMOIRE (son + // `block` est `None`, personne d'autre ne la liberera). D'ou : relacher + // le wrapper d'abord, liberer la memoire ensuite. + drop(self.buffer.take()); + unsafe { + self.device.free_memory(self.memory, None); + } + // Le fd est un handle a part : l'exporter duplique la propriete, donc le + // fermer ne libere pas la memoire -- mais l'oublier fuirait un + // descripteur par frame. + if self.fd >= 0 { + let _ = nix_close(self.fd); + } + } +} + +fn nix_close(fd: i32) -> std::io::Result<()> { + // `libc::close` sans dependance supplementaire : la libc est deja liee. + extern "C" { + fn close(fd: i32) -> i32; + } + if unsafe { close(fd) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +impl Compositor { + /// Alloue un buffer de staging exportable de `size` octets, ou `None` si le + /// device n'a pas ete ouvert avec les extensions de memoire externe (cf. + /// `d3d_linux::open_device_with_dmabuf_export`). + pub fn create_exportable_staging(&self, size: u64) -> Option { + use ash::vk; + unsafe { + self.gpu.device.as_hal::(|hal| { + let hal = hal?; + let dev = hal.raw_device().clone(); + let phys = hal.raw_physical_device(); + let instance = hal.shared_instance().raw_instance(); + + let mut ext_info = vk::ExternalMemoryBufferCreateInfo::default() + .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let bci = vk::BufferCreateInfo::default() + .push_next(&mut ext_info) + .size(size) + .usage(vk::BufferUsageFlags::TRANSFER_DST) + .sharing_mode(vk::SharingMode::EXCLUSIVE); + let raw = dev.create_buffer(&bci, None).ok()?; + + let req = dev.get_buffer_memory_requirements(raw); + let props = instance.get_physical_device_memory_properties(phys); + // HOST_VISIBLE pour que `read_back` puisse verifier le contenu, + // et COHERENT parce qu'il lit SANS invalider : sur une memoire + // seulement visible, le mapping peut rendre des octets perimes et + // le test passerait ou echouerait selon le cache, pas selon le + // code. Exiger les deux est plus simple qu'un + // `vkInvalidateMappedMemoryRanges` correct a chaque lecture. + let want = vk::MemoryPropertyFlags::HOST_VISIBLE + | vk::MemoryPropertyFlags::HOST_COHERENT; + let mt = (0..props.memory_type_count).find(|i| { + req.memory_type_bits & (1 << i) != 0 + && props.memory_types[*i as usize].property_flags.contains(want) + })?; + + let mut export = vk::ExportMemoryAllocateInfo::default() + .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let mai = vk::MemoryAllocateInfo::default() + .push_next(&mut export) + .allocation_size(req.size) + .memory_type_index(mt); + let memory = dev.allocate_memory(&mai, None).ok()?; + dev.bind_buffer_memory(raw, memory, 0).ok()?; + + let getter = ash::khr::external_memory_fd::Device::new(instance, &dev); + let fd = getter + .get_memory_fd( + &vk::MemoryGetFdInfoKHR::default() + .memory(memory) + .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT), + ) + .ok()?; + + let hal_buf = wgpu_hal::vulkan::Device::buffer_from_raw(raw); + let buffer = self.gpu.device.create_buffer_from_hal::( + hal_buf, + &wgpu::BufferDescriptor { + label: Some("staging-exportable"), + size, + usage: wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }, + ); + Some(ExportableStaging { buffer: Some(buffer), fd, size, device: dev, memory }) + }) + } + } +} + +impl Compositor { + /// Compose la frame courante en NV12 et la depose dans `staging`, dont la + /// memoire est exportable en dmabuf. Rend la main quand le GPU a fini. + /// + /// PAS DE RING, PAS DE `map_async`, CONTRAIREMENT A `readback_submit_yuv`. + /// Cette variante-ci n'a rien a faire relire par le CPU : le consommateur est + /// l'encodeur materiel, qui lit la meme memoire par son fd. Toute la + /// mecanique de staging mappe et de recolte differee n'aurait donc personne a + /// servir. + /// + /// NE BLOQUE PAS. Rend l'index de soumission ; l'appelant attend dessus juste + /// avant de donner le fd a l'encodeur, ce qui lui laisse la fenetre pour + /// composer la frame suivante pendant que celle-ci finit. C'est le meme + /// pipelining que la ring de relecture software, avec des tampons + /// exportables a la place des buffers mappes. + pub unsafe fn compose_into_dmabuf( + &self, + staging: &ExportableStaging, + ) -> Result { + self.ensure_yuv_fmt(YuvFormat::Nv12)?; + let (bpr_y, bpr_uv, off_uv, total) = { + let g = self.yuv.borrow(); + let t = g.as_ref().expect("ensure_yuv"); + (t.bpr_y, t.bpr_uv, t.off_u, t.total) + }; + if staging.size < total { + anyhow::bail!("staging de {} octets pour {total} attendus", staging.size); + } + let (w, h) = (self.render_w, self.render_h); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + + let mut encoder = self + .gpu + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("yuv-dmabuf") }); + { + let g = self.yuv.borrow(); + let t = g.as_ref().expect("ensure_yuv"); + let (uv_view, pipe_uv, _uv) = match &t.chroma { + Chroma::Interleaved { uv_view, pipe_uv, _uv } => (uv_view, pipe_uv, _uv), + Chroma::Planar { .. } => { + anyhow::bail!("compose_into_dmabuf attend des cibles NV12") + } + }; + for (view, pipe) in [(&t.y_view, &t.pipe_y), (uv_view, pipe_uv)] { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("yuv-dmabuf-plane"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(pipe); + pass.set_bind_group(0, &t.bind, &[]); + pass.draw(0..3, 0..1); + } + for (tex, off, bpr, pw, ph) in + [(&t._y, 0u64, bpr_y, w, h), (_uv, off_uv, bpr_uv, cw, ch)] + { + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: staging.buffer(), + layout: wgpu::TexelCopyBufferLayout { + offset: off, + bytes_per_row: Some(bpr), + rows_per_image: Some(ph), + }, + }, + wgpu::Extent3d { width: pw, height: ph, depth_or_array_layers: 1 }, + ); + } + } + Ok(self.gpu.context.submit(std::iter::once(encoder.finish()))) + } + + /// Attend qu'une soumission soit terminee. + /// + /// INDISPENSABLE AVANT DE PASSER LE FD. L'encodeur lit cette memoire par un + /// chemin que wgpu ignore : rien d'autre ne garantirait que la copie a bien + /// atterri. + pub fn wait_submission(&self, idx: wgpu::SubmissionIndex) { + self.gpu.device.poll(wgpu::Maintain::WaitForSubmissionIndex(idx)); + } + + /// La geometrie NV12 courante, pour decrire le dmabuf au consommateur. + pub fn nv12_geometry(&self) -> (u32, u32, u64, u64) { + Compositor::yuv_layout_for(self.render_w, self.render_h, YuvFormat::Nv12) + } +} diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index cffc9395d..e3c8cff5e 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -42,6 +42,15 @@ use anyhow::{anyhow, Result}; use metal::foreign_types::ForeignType; use std::cell::RefCell; +/// Budget du cache de textures image (`img_cache`), en octets. Même valeur et même raison que +/// `compositor_windows::IMG_CACHE_BUDGET_BYTES`. +/// +/// Doit tenir le JEU ACTIF d'une frame — au pire un wallpaper d'écran ET un fond de caméra, que +/// rien n'empêche d'être deux 7680x7680 à 225 Mo pièce. Sous ce seuil l'éviction ne peut plus +/// rendre de mémoire sans toucher au jeu actif, ce qu'elle refuse de faire. 512 Mo borne la fuite +/// (1 774 Mo mesurés en parcourant les 18 wallpapers livrés) en laissant le jeu actif résident. +const IMG_CACHE_BUDGET_BYTES: u64 = 512 * 1024 * 1024; + // --------------------------------------------------------------------------- // CVMetalTextureCache — le pont CVPixelBuffer → MTLTexture // --------------------------------------------------------------------------- @@ -197,6 +206,42 @@ impl Drop for CVMetalTextureCache { } } +// --------------------------------------------------------------------------- +// Segmentation du sujet webcam +// --------------------------------------------------------------------------- + +/// Cadence de l'inférence. Même valeur et même raison que +/// `compositor_windows::SEGMENTATION_HZ` : une silhouette ne bouge pas de façon +/// perceptible en 16 ms, et c'est le seul levier mesuré qui divise le coût par deux sans +/// toucher au modèle. +const SEGMENTATION_HZ: u32 = 30; + +/// Cible RGBA + miroir de lecture pour extraire la frame webcam à la résolution du modèle. +/// +/// Deux textures, pas une : `rt` est `Private` parce que c'est une cible de rendu, et +/// `get_bytes` n'est légal que sur du `Shared`. C'est exactement le couple +/// `nv12_y`/`nv12_read_y` du chemin d'encodage, en RGBA et à 256x144 — cf. l'en-tête du +/// module. `Managed` n'a pas sa place ici : rien dans ce fichier n'en utilise, et c'est le +/// seul mode de stockage qui exigerait un `synchronizeResource` avant la lecture. +struct SegCapture { + /// Cible de la passe de capture. `Private` : écrite par le GPU, jamais lue par le CPU. + rt: metal::Texture, + /// Miroir `Shared` de `rt`, rempli par blit dans le même command buffer. + read: metal::Texture, + width: u32, + height: u32, +} + +/// Texture du masque de segmentation, recréée seulement quand la résolution du modèle +/// change — c'est-à-dire jamais, en régime établi. Pendant Metal de +/// `compositor_windows::WebcamMask` : pas de vue à côté de la texture, un `MTLTexture` est +/// déjà ce que `set_fragment_texture` prend. +struct WebcamMask { + tex: metal::Texture, + width: u32, + height: u32, +} + // --------------------------------------------------------------------------- // Compositor // --------------------------------------------------------------------------- @@ -226,8 +271,15 @@ pub struct Compositor { last_cmd: RefCell>, /// Wallpapers décodés, indexés par chemin (ou par data-URI pour les annotations image). /// Le décode + upload coûte des millisecondes ; le faire à chaque frame ferait chuter la - /// preview sur un fond image. - img_cache: RefCell>, + /// preview sur un fond image. L'entrée reste néanmoins évinçable dès qu'elle sort du jeu + /// actif d'une frame — cf. `cached_image`. + img_cache: RefCell>, + /// Compteur d'accès de `img_cache`, pour l'ordre LRU. Un compteur plutôt que l'index de + /// frame : une frame touche plusieurs entrées, et il faut pouvoir les ordonner entre elles. + img_tick: std::cell::Cell, + /// Valeur de `img_tick` au début de la frame en cours. Tout ce qui a été touché depuis + /// appartient au jeu actif et ne peut pas être évincé — voir `cached_image`. + img_frame_start: std::cell::Cell, // --- Engine : render targets --- /// Render target principal RGBA8. Cible de `compose_frame`. `Private` : c'est une @@ -280,6 +332,30 @@ pub struct Compositor { /// Textes rastérisés, indexés par ID, avec la `cache_key` du spec pour invalider. text_cache: RefCell>, text_raster: Option, + + // --- Segmentation du sujet webcam (cf. `pump_segmentation`) --- + /// Masque du sujet, R8 à la résolution du modèle. Écrit par `set_webcam_mask`, lu au + /// moment de dessiner la webcam. `None` tant qu'aucune frame n'a été segmentée — l'effet + /// reste alors éteint plutôt que de rendre une webcam invisible en mode détourage. + webcam_mask: RefCell>, + /// Cible + miroir de la capture, créés à la première capture et jamais redimensionnés : + /// le modèle a une entrée fixe. + seg_capture: RefCell>, + /// Worker d'inférence, absent tant que `enable_segmentation` n'a pas été appelé. + seg_worker: RefCell>, + /// Segmenteur tenu SUR LE THREAD DE RENDU, utilisé à la place du worker en mode + /// déterministe. Voir `set_segmentation_deterministic`. + seg_sync: RefCell>, + /// Export : cadence par frame et inférence synchrone, au lieu de l'horloge et du worker. + seg_deterministic: std::cell::Cell, + /// Boîte aux lettres du worker. Le masque est déposé depuis le thread d'inférence et + /// téléversé depuis le thread de rendu : aucun appel Metal ne traverse de thread. + seg_inbox: std::sync::Arc>>>, + seg_rate: RefCell, + /// Frame RGB réutilisée d'une capture à l'autre. + seg_scratch: RefCell>, + /// Le chargement du modèle a échoué : ne pas réessayer à chaque frame. + seg_failed: RefCell, } /// Descripteur de texture — les six cibles ne diffèrent que par format, taille et @@ -539,6 +615,8 @@ impl Compositor { metal_texture_cache: cache, last_cmd: RefCell::new(None), img_cache: RefCell::new(std::collections::HashMap::new()), + img_tick: std::cell::Cell::new(0), + img_frame_start: std::cell::Cell::new(0), rt, rt_read, nv12_y, @@ -561,6 +639,15 @@ impl Compositor { ann_img_cache: RefCell::new(std::collections::HashMap::new()), text_cache: RefCell::new(std::collections::HashMap::new()), text_raster: crate::text::TextRasterizer::new().ok(), + webcam_mask: RefCell::new(None), + seg_capture: RefCell::new(None), + seg_worker: RefCell::new(None), + seg_sync: RefCell::new(None), + seg_deterministic: std::cell::Cell::new(false), + seg_inbox: std::sync::Arc::new(std::sync::Mutex::new(None)), + seg_rate: RefCell::new(crate::segmentation::RateLimiter::new(SEGMENTATION_HZ)), + seg_scratch: RefCell::new(Vec::new()), + seg_failed: RefCell::new(false), }) } @@ -783,6 +870,54 @@ impl Compositor { Ok((tex, w, h)) } + /// Ouvre une frame du point de vue de `img_cache` : tout ce qui sera touché après cet appel + /// est le jeu actif, et devient inévinçable jusqu'à la frame suivante. + fn begin_image_frame(&self) { + // `+ 1` : la première entrée de cette frame recevra `img_tick + 1`, et la protection + // porte sur `tick >= img_frame_start`. Sans le décalage on protégerait aussi la + // DERNIÈRE entrée de la frame précédente, qui n'appartient plus au jeu actif — le + // résident pourrait alors dépasser le budget d'une texture entière. + self.img_frame_start.set(self.img_tick.get() + 1); + } + + /// Texture d'un fichier image, décodée une seule fois puis réutilisée. + /// + /// Le cache était NON BORNÉ, et c'est un vrai coût : les wallpapers livrés pèsent 23,7 Mo sur + /// disque mais 1 774 Mo une fois décodés en RGBA8 — `wallpaper8.jpg` fait 7680x7680, soit + /// 225 Mo à lui seul. Parcourir le sélecteur les chargeait tous et n'en libérait aucun. + /// + /// L'éviction est LRU sous un budget en octets, et ne touche jamais une texture que la frame + /// EN COURS a déjà servie : sans ça, un fond d'écran et un fond de caméra un peu gros se + /// chasseraient l'un l'autre à chaque frame, et un décodage coûte 129 ms contre les ~3,5 ms + /// d'une frame. Si le jeu actif dépasse à lui seul le budget, on dépasse le budget. + fn cached_image(&self, path: &str) -> Result<(metal::Texture, u32, u32)> { + let tick = self.img_tick.get() + 1; + self.img_tick.set(tick); + // Emprunt isolé dans un `let` pour qu'il soit relâché AVANT le `borrow_mut` — + // même piège que côté Windows (double emprunt RefCell à la première frame image). + let hit = self.img_cache.borrow().get(path).cloned(); + if let Some((tex, w, h, _)) = hit { + self.img_cache.borrow_mut().insert(path.to_string(), (tex.clone(), w, h, tick)); + return Ok((tex, w, h)); + } + let (tex, w, h) = self.load_image_texture(path)?; + let mut cache = self.img_cache.borrow_mut(); + cache.insert(path.to_string(), (tex.clone(), w, h, tick)); + // La politique vit dans `frame_geometry` : les trois backends la partagent, comme la + // géométrie, plutôt que d'entretenir trois copies qui finiraient par diverger. + let entries: Vec<(String, u64, u64)> = cache + .iter() + .map(|(k, e)| (k.clone(), e.1 as u64 * e.2 as u64 * 4, e.3)) + .collect(); + let protect_from = self.img_frame_start.get(); + for key in + crate::frame_geometry::lru_evictions(&entries, IMG_CACHE_BUDGET_BYTES, protect_from) + { + cache.remove(&key); + } + Ok((tex, w, h)) + } + /// Fond wallpaper image, cover-fit sur le ratio de SORTIE (mode 6). /// /// Le crop de recouvrement se calcule contre le vrai ratio de sortie, pas contre celui @@ -793,17 +928,23 @@ impl Compositor { path: &str, output_aspect: f32, ) -> Result<()> { - // Emprunt isolé dans un `let` pour qu'il soit relâché AVANT le `borrow_mut` — - // même piège que côté Windows (double emprunt RefCell à la première frame image). - let cached = self.img_cache.borrow().get(path).cloned(); - let (tex, iw, ih) = match cached { - Some(v) => v, - None => { - let loaded = self.load_image_texture(path)?; - self.img_cache.borrow_mut().insert(path.to_string(), loaded.clone()); - loaded - } - }; + self.draw_image_in(enc, path, [0.0, 0.0, 1.0, 1.0], [0.0, 0.0], 0.0, output_aspect) + } + + /// `draw_image_bg` pour un rect quelconque — la bulle webcam s'en sert avec ses coins + /// arrondis. `output_aspect` est le ratio du RECT visé, pas celui de la sortie : le crop + /// « cover » se calcule contre la zone qu'on remplit. + #[allow(clippy::too_many_arguments)] + unsafe fn draw_image_in( + &self, + enc: &metal::RenderCommandEncoderRef, + path: &str, + dst: [f32; 4], + quad_px: [f32; 2], + radius_px: f32, + output_aspect: f32, + ) -> Result<()> { + let (tex, iw, ih) = self.cached_image(path)?; let ai = iw as f32 / ih.max(1) as f32; let ao = output_aspect; let (u0, v0, u1, v1) = if ai > ao { @@ -817,8 +958,10 @@ impl Compositor { self.draw_solid( enc, &LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], + dst, src: [u0, v0, u1, v1], + quad_px, + radius_px, mode: 6.0, ..Default::default() }, @@ -826,7 +969,72 @@ impl Compositor { Ok(()) } - + /// Peint le fond du mode « personnalisé » DANS la bulle webcam, avant que la caméra n'y soit + /// découpée par-dessus. + /// + /// Le shader ne sait peindre qu'une couleur plate sous le masque, donc un dégradé ou une + /// image y tombaient sur du noir — et le défaut EST une image (`DEFAULT_WALLPAPER`), si bien + /// que le mode ne rendait jamais ce que le sélecteur montrait. Peindre le fond puis composer + /// la caméra en détourage donne exactement le même résultat (`lerp(fond, caméra, personne)`, + /// ici par le mélange alpha) pour les trois sortes de fond, en réutilisant les chemins déjà + /// éprouvés du fond d'écran, et sans rien ajouter aux trois shaders. + /// + /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit épouser ses coins arrondis, + /// sinon un rectangle déborde derrière la caméra. + unsafe fn draw_webcam_bg( + &self, + enc: &metal::RenderCommandEncoderRef, + bg: Option<&SceneBackground>, + dst: [f32; 4], + quad_px: [f32; 2], + radius_px: f32, + ) { + const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + let solid = |color: [f32; 4]| LayerCB { + dst, + quad_px, + radius_px, + mode: 1.0, + color, + ..Default::default() + }; + match bg { + Some(SceneBackground::Color { color }) => { + self.draw_solid(enc, &solid(parse_hex(color).unwrap_or(BLACK))); + } + Some(SceneBackground::Gradient { angle_deg, stops }) => { + let c0 = stops.first().and_then(|s| parse_hex(s)).unwrap_or(BLACK); + let c1 = stops.last().and_then(|s| parse_hex(s)).unwrap_or(c0); + // angle CSS → direction unitaire, même convention que le fond d'écran. + let a = angle_deg.to_radians(); + self.draw_solid( + enc, + &LayerCB { + dst, + src: [c1[0], c1[1], c1[2], c1[3]], + quad_px, + radius_px, + mode: 5.0, + color: c0, + fx: [a.sin(), -a.cos(), 0.0, 0.0], + ..Default::default() + }, + ); + } + Some(SceneBackground::Image { path }) => { + // Même contrat que le fond d'écran : un chemin cassé est loggé puis remplacé par + // du noir. Un fallback silencieux redonnerait le bug qu'on corrige. + let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; + if let Err(e) = self.draw_image_in(enc, path, dst, quad_px, radius_px, aspect) { + eprintln!("[compositor] fond webcam \"{path}\" : {e:#}"); + self.draw_solid(enc, &solid(BLACK)); + } + } + // Personnalisé sans fond : noir, comme avant — mais c'est désormais le seul chemin + // qui y mène, au lieu de l'être pour toute image et tout dégradé. + None => self.draw_solid(enc, &solid(BLACK)), + } + } /// Une passe plein écran : `source` -> `target` avec `pipeline`, `fx` dans le LayerCB. /// Le viewport découle de la taille de l'attachement, donc pas de `RSSetViewports`. @@ -1208,6 +1416,363 @@ impl Compositor { } + /// Extrait la frame webcam en RGB8 à la résolution du modèle, dans `out`. + /// + /// Pendant Metal de `compositor_windows::capture_webcam_rgb`, avec les mêmes contraintes + /// d'appel et une seule divergence de mécanique : là où D3D11 réquisitionne la cible du + /// contexte persistant, Metal ouvre une passe sur `SegCapture::rt` et la referme, donc + /// rien n'est « réquisitionné ». La contrainte d'ordre reste malgré tout : cette méthode + /// **doit tourner avant que le command buffer de composition ne soit créé**, parce + /// qu'elle attend son propre buffer et qu'attendre au milieu d'une frame sérialiserait + /// CPU et GPU sur exactement le chemin que cette conception veut garder recouvert. + /// + /// `src` est le rect source en UV. L'appelant y passe la frame ENTIÈRE et non le + /// sous-rect dessiné — cf. `pump_segmentation`. + /// + /// # Le readback + /// + /// Trois étapes, la forme prescrite par l'en-tête du module et déjà tenue par + /// `render_nv12` + `read_nv12_scaled` : rendu dans une cible `Private`, blit vers un + /// miroir `Shared`, `get_bytes`. Pas de `Managed`, donc pas de `synchronizeResource` — + /// c'est le seul mode de stockage qui l'exigerait, et rien dans ce fichier n'en utilise. + /// + /// Le buffer `out` est réutilisé d'un appel à l'autre : il est dimensionné au RGBA lu + /// puis compacté sur place en RGB, ce qui laisse sa capacité au maximum des deux et ne + /// réalloue donc plus après la première capture. + pub unsafe fn capture_webcam_rgb( + &self, + wy: &metal::Texture, + wuv: &metal::Texture, + src: [f32; 4], + width: u32, + height: u32, + out: &mut Vec, + ) -> Result<()> { + if width == 0 || height == 0 { + return Err(anyhow!( + "capture webcam de dimensions nulles ({width}x{height})" + )); + } + { + let mut slot = self.seg_capture.borrow_mut(); + if !matches!(slot.as_ref(), Some(c) if c.width == width && c.height == height) { + *slot = Some(SegCapture { + rt: make_texture( + &self.gpu.device, + metal::MTLPixelFormat::RGBA8Unorm, + width, + height, + metal::MTLStorageMode::Private, + metal::MTLTextureUsage::RenderTarget | metal::MTLTextureUsage::ShaderRead, + ), + read: make_texture( + &self.gpu.device, + metal::MTLPixelFormat::RGBA8Unorm, + width, + height, + metal::MTLStorageMode::Shared, + metal::MTLTextureUsage::ShaderRead, + ), + width, + height, + }); + } + } + let slot = self.seg_capture.borrow(); + let cap = slot.as_ref().expect("créé juste au-dessus"); + + // Command buffer PROPRE, et surtout PAS `submit`/`sync` : `sync` attend `last_cmd`, + // et `read_nv12_scaled` compte sur `last_cmd` pour être le buffer de `render_nv12`. + // Le remplacer ici ferait attendre la capture au lieu de la conversion NV12, et le + // readback d'encodage lirait des plans que rien n'a encore écrits. + let cmd_buf = self.gpu.context.new_command_buffer(); + { + // Plein cadre de la cible, sans coins ni motion blur : le modèle veut l'image, + // pas la mise en forme. `fx` reste à zéro — la branche de masque du shader ne + // doit surtout pas se prendre sur la capture qui l'alimente. + let enc = self.begin_pass( + cmd_buf, + &cap.rt, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), + &self.pipeline_main, + )?; + self.draw_video( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src, + quad_px: [width as f32, height as f32], + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + mb: [1.0, 1.0, 1.0, 0.0], + ..Default::default() + }, + wy, + wuv, + ); + enc.end_encoding(); + } + let blit = cmd_buf.new_blit_command_encoder(); + blit.copy_from_texture( + &cap.rt, + 0, + 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize { width: width as u64, height: height as u64, depth: 1 }, + &cap.read, + 0, + 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + ); + blit.end_encoding(); + cmd_buf.commit(); + cmd_buf.wait_until_completed(); + + let (w, h) = (width as usize, height as usize); + out.resize(w * h * 4, 0); + cap.read.get_bytes( + out.as_mut_ptr() as *mut std::ffi::c_void, + (w * 4) as u64, + metal::MTLRegion { + origin: metal::MTLOrigin { x: 0, y: 0, z: 0 }, + size: metal::MTLSize { width: w as u64, height: h as u64, depth: 1 }, + }, + 0, + ); + // RGBA → RGB sur place : le modèle n'a pas de canal alpha en entrée. La destination + // (`3i`) court derrière la source (`4i`), donc aucune écriture n'écrase un octet pas + // encore lu. + for i in 0..w * h { + let (r, g, b) = (out[i * 4], out[i * 4 + 1], out[i * 4 + 2]); + out[i * 3] = r; + out[i * 3 + 1] = g; + out[i * 3 + 2] = b; + } + out.truncate(w * h * 3); + Ok(()) + } + + /// Publie le masque de segmentation du sujet webcam (R8, `width`x`height`, 0 = fond). + /// + /// La texture est `Shared` et réécrite en place par `replace_region` ; elle n'est + /// recréée que si la résolution du modèle change, ce qui n'arrive pas en régime établi. + /// + /// Réécrire une texture que le GPU pourrait encore lire serait une course — ici il ne + /// le peut pas : les trois chemins de frame macOS drainent la file avant de rendre la + /// main (`readback_direct` et `rgb_to_nv12` font `submit` + `sync`, `read_nv12_scaled` + /// fait `sync`), donc plus rien n'est en vol quand `compose_frame` rappelle + /// `pump_segmentation`. C'est ce qui dispense d'un double buffer, pas la chance. + pub fn set_webcam_mask(&self, data: &[u8], width: u32, height: u32) -> Result<()> { + if width == 0 || height == 0 { + return Err(anyhow!("masque webcam de dimensions nulles ({width}x{height})")); + } + let expected = (width as usize) * (height as usize); + if data.len() < expected { + return Err(anyhow!( + "masque webcam trop court : {} octets pour {width}x{height}", + data.len() + )); + } + + let mut slot = self.webcam_mask.borrow_mut(); + if !matches!(slot.as_ref(), Some(m) if m.width == width && m.height == height) { + *slot = Some(WebcamMask { + tex: make_texture( + &self.gpu.device, + metal::MTLPixelFormat::R8Unorm, + width, + height, + metal::MTLStorageMode::Shared, + metal::MTLTextureUsage::ShaderRead, + ), + width, + height, + }); + } + let mask = slot.as_ref().expect("alloué juste au-dessus"); + mask.tex.replace_region( + metal::MTLRegion { + origin: metal::MTLOrigin { x: 0, y: 0, z: 0 }, + size: metal::MTLSize { width: width as u64, height: height as u64, depth: 1 }, + }, + 0, + data.as_ptr() as *const std::ffi::c_void, + width as u64, + ); + Ok(()) + } + + /// Un tour de segmentation : téléverse le masque prêt, puis soumet une nouvelle frame si + /// la cadence l'autorise. Port de `compositor_windows::pump_segmentation` — worker, + /// boîte aux lettres, limiteur de cadence et démarrage paresseux sont indépendants de la + /// plateforme, seuls les deux appels GPU changent. + /// + /// Les deux moitiés sont volontairement désynchronisées. Le masque téléversé ici vient de + /// la frame précédente — une frame de retard sur une silhouette est invisible, alors + /// qu'attendre l'inférence bloquerait le rendu, ce qui est exactement le coût que toute + /// cette conception cherche à ne pas payer. + unsafe fn pump_segmentation( + &self, + wy: &metal::Texture, + wuv: &metal::Texture, + valid: [f32; 2], + ) -> Result<()> { + if *self.seg_failed.borrow() { + return Ok(()); + } + // Rien à faire si aucun effet n'est demandé : ni capture, ni inférence, ni masque. + // Le coût de la fonctionnalité est alors exactement nul. + let (wants_effect, model_path) = { + let scene = self.scene.borrow(); + match scene.as_ref().and_then(|s| s.webcam_effect.as_ref()) { + Some(e) if e.shader_code() > 0.0 => (true, e.model_path.clone()), + _ => (false, None), + } + }; + if !wants_effect { + return Ok(()); + } + + // Démarrage paresseux, piloté par la scène : personne n'a à appeler + // `enable_segmentation` à la main, et un modèle introuvable éteint l'effet au lieu + // de faire tomber le rendu. + if self.seg_worker.borrow().is_none() && self.seg_sync.borrow().is_none() { + let Some(path) = model_path else { return Ok(()) }; + if let Err(e) = self.enable_segmentation(std::path::Path::new(&path)) { + eprintln!("[segmentation] désactivée : {e}"); + // Une scène qui reste identique retenterait à chaque frame ; on lève le + // verrou plutôt que de journaliser 60 fois par seconde. + *self.seg_failed.borrow_mut() = true; + return Ok(()); + } + // En preview on rend cette frame sans masque : le worker vient de démarrer et + // l'effet apparaîtra dans quelques millisecondes, ce que personne ne voit. À + // l'export cette frame part dans le fichier — on enchaîne donc sur la capture et + // l'inférence plutôt que de la laisser sortir non détourée. + if !self.seg_deterministic.get() { + return Ok(()); + } + } + + if let Some(mask) = self.seg_inbox.lock().unwrap().take() { + self.set_webcam_mask( + &mask, + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + )?; + } + + // La cadence horloge est le bon réglage en preview et le mauvais à l'export, où les + // frames défilent aussi vite que la machine décode : le nombre de frames couvertes par + // un masque dépendrait alors de la charge. En déterministe, une inférence par frame. + if !self.seg_deterministic.get() + && !self.seg_rate.borrow_mut().should_run(std::time::Instant::now()) + { + return Ok(()); + } + let mut scratch = self.seg_scratch.borrow_mut(); + // La frame ENTIÈRE, pas le sous-rect dessiné : un crop utilisateur serré amputerait + // le sujet en entrée du modèle, et le masque serait faux là où il compte le plus. + // Le shader ramène ses coordonnées dans cet espace via `fx.xy`. + self.capture_webcam_rgb( + wy, + wuv, + [0.0, 0.0, valid[0], valid[1]], + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + &mut scratch, + )?; + if self.seg_deterministic.get() { + // Synchrone : le masque doit exister avant que cette frame ne soit composée, sinon + // on retombe sur le défaut qu'on corrige. Une inférence ratée laisse le masque + // précédent, comme le fait le worker. + let mut sync = self.seg_sync.borrow_mut(); + if let Some(seg) = sync.as_mut() { + match seg.run(&scratch) { + Ok(mask) => { + let mask = mask.to_vec(); + drop(sync); + self.set_webcam_mask( + &mask, + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + )?; + } + Err(e) => eprintln!("[segmentation] frame ignorée : {e}"), + } + } + } else if let Some(w) = self.seg_worker.borrow().as_ref() { + w.submit(&scratch); + } + Ok(()) + } + + /// Démarre la segmentation du sujet webcam pour ce compositeur. + /// + /// Idempotent. Tant qu'elle n'est pas appelée, `compose_frame` ne fait rien de plus et + /// la webcam se dessine comme avant — c'est ce qui rend l'effet inerte plutôt que cassé + /// sur une build sans modèle. + pub fn enable_segmentation(&self, model_path: &std::path::Path) -> Result<()> { + if self.seg_worker.borrow().is_some() || self.seg_sync.borrow().is_some() { + return Ok(()); + } + let segmenter = crate::segmentation::Segmenter::load(model_path)?; + // En déterministe, le segmenteur reste ici : l'inférence tourne sur le thread de rendu, + // donc le masque de la frame N est prêt AVANT qu'elle ne soit composée. Le worker est un + // choix de preview — ne jamais bloquer l'affichage — et c'est exactement ce qui rend + // l'export irreproductible, le masque arrivant quelques frames plus tard selon la charge. + if self.seg_deterministic.get() { + *self.seg_sync.borrow_mut() = Some(segmenter); + return Ok(()); + } + let inbox = std::sync::Arc::clone(&self.seg_inbox); + let worker = crate::segmentation::SegmentationWorker::spawn(segmenter, move |mask, _, _| { + // Écrase le masque précédent s'il n'a pas encore été téléversé : c'est le plus + // récent qui vaut, jamais une file. + *inbox.lock().unwrap() = Some(mask.to_vec()); + }); + *self.seg_worker.borrow_mut() = Some(worker); + Ok(()) + } + + /// Bascule la segmentation en mode reproductible, pour l'export. + /// + /// En preview, la cadence suit l'horloge (30 Hz réels) et l'inférence tourne sur un worker : + /// c'est le bon choix, l'affichage ne doit jamais attendre. À l'export les frames sont rendues + /// aussi vite que la machine décode, sans rapport avec le temps réel — et ces deux choix + /// deviennent alors des bugs. La cadence horloge fait dépendre le nombre de frames couvertes + /// par un masque de la vitesse de la machine, et le worker asynchrone rend les premières + /// frames AVANT que le premier masque n'existe : elles partent dans le fichier avec le vrai + /// arrière-plan de la webcam. Deux exports du même projet ne donnent donc pas les mêmes + /// pixels, ce qui casse l'invariant « l'export est identique à la preview ». + /// + /// En déterministe : une inférence PAR FRAME, synchrone. Plus coûteux (~3 ms/frame), mais + /// l'export est hors ligne et chaque frame porte le masque calculé depuis SA propre image. + /// + /// À appeler avant la première frame — c'est ce qui décide comment `enable_segmentation` + /// s'installe. + pub fn set_segmentation_deterministic(&self, on: bool) { + if self.seg_deterministic.get() == on { + return; + } + self.seg_deterministic.set(on); + // Changer de mode change le MOTEUR, et `enable_segmentation` est idempotent sur la + // PRÉSENCE d'un moteur : sans démonter celui qui ne correspond plus, le drapeau mentirait. + // Un compositeur qui a déjà servi en preview garderait son worker, `seg_sync` resterait + // vide, et l'export entier ne ferait AUCUNE inférence. Le démarrage paresseux de + // `pump_segmentation` réinstalle le bon moteur à la frame suivante. + *self.seg_worker.borrow_mut() = None; + *self.seg_sync.borrow_mut() = None; + // Et le masque que le worker démonté avait peut-être déjà déposé : il vient de l'autre + // mode, il n'a rien à faire sur la première frame de celui-ci. + *self.seg_inbox.lock().unwrap() = None; + } + + /// Éteint l'effet : la webcam se redessine telle quelle à la frame suivante. + pub fn clear_webcam_mask(&self) { + *self.webcam_mask.borrow_mut() = None; + } + /// Soumet sans attendre, et retient le buffer pour `sync`. fn submit(&self, cmd: &metal::CommandBufferRef) { cmd.commit(); @@ -1264,15 +1829,7 @@ impl Compositor { sprite: &crate::scene::SceneCursorSprite, clip: [f32; 4], ) -> Result<()> { - let cached = self.img_cache.borrow().get(sprite.path.as_str()).cloned(); - let (tex, iw, ih) = match cached { - Some(v) => v, - None => { - let loaded = self.load_image_texture(&sprite.path)?; - self.img_cache.borrow_mut().insert(sprite.path.clone(), loaded.clone()); - loaded - } - }; + let (tex, iw, ih) = self.cached_image(sprite.path.as_str())?; let (rw, rh) = (self.render_w as f32, self.render_h as f32); let ar = iw as f32 / ih.max(1) as f32; let (pw, ph) = if ar >= 1.0 { (size_px, size_px / ar) } else { (size_px * ar, size_px) }; @@ -1373,6 +1930,7 @@ impl Compositor { frame: f32, cfg: &Cfg, ) -> Result<()> { + self.begin_image_frame(); if Self::pixel_buffer_of(screen).is_none() { return self.clear_rt(); } @@ -1391,6 +1949,19 @@ impl Compositor { let u_max = scw / (stw.max(1)) as f32; let v_max = sch / (sth.max(1)) as f32; let (rw, rh) = (self.render_w as f32, self.render_h as f32); + // Étendue valide de la texture webcam : les décodeurs allouent des textures alignées, + // donc la frame n'occupe pas forcément toute la texture. `.max(1)` au dénominateur — + // `tex_dims` rend (0, 0) sur une webcam absente, là où le chemin Windows divise sans + // garde parce qu'il a toujours les deux frames. + let w_valid = [wcw / (wtw.max(1)) as f32, wch / (wth.max(1)) as f32]; + + // Segmentation, AVANT d'ouvrir le command buffer de composition : `capture_webcam_rgb` + // attend son propre buffer, et attendre au milieu de la frame sérialiserait CPU et GPU. + // Dernier point où `wtw/wth/wcw/wch` sont en portée sans emprunt de `self.scene` — + // `pump_segmentation` emprunte la scène lui-même. + if let Some((wy, wuv)) = webcam_tex.as_ref() { + self.pump_segmentation(wy, wuv, w_valid)?; + } let scene_ref = self.scene.borrow(); let cursor_ref = self.cursor.borrow(); @@ -1531,7 +2102,7 @@ impl Compositor { color: [0.0, 0.0, 0.0, 1.0], src_prev: [su0, sv0, su1, sv1], dst_prev: g.s_dst_prev, - mb: [g.mb_taps, 1.0, 1.0, 0.0], + mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], ..Default::default() }, &sy, @@ -1580,10 +2151,10 @@ impl Compositor { Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 0.0)), &self.pipeline_add, )?; - let w = 1.0 / plan.taps as f32; - e.set_blend_color(w, w, w, w); for k in 0..plan.taps { let f = k as f32 / (plan.taps - 1) as f32; + let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); + e.set_blend_color(w, w, w, w); self.draw_cur_themed( e, &sprites, @@ -1618,7 +2189,25 @@ impl Compositor { g.scene_preset.as_deref(), Some("dual-frame") | Some("vertical-stack") ); - if cfg.shadow && !webcam_is_block && g.shape_fade > 0.0 { + // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de + // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam + // invisible en détourage, donc tant que rien n'a été segmenté on dessine la + // piste telle quelle. C'est aussi ce qui rend le premier lancement gracieux. + let mask = self.webcam_mask.borrow(); + let effect = scene_ref + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .filter(|_| mask.is_some()) + .map(|e| (e.shader_code(), e)) + .filter(|(code, _)| *code > 0.0); + + // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une + // ombre portée par un rectangle invisible se lit comme un artefact. Le test porte + // sur le code de la SCÈNE et non sur celui envoyé au shader : le fond personnalisé + // part lui aussi en détourage ci-dessous, mais sa bulle, elle, est bien peinte et + // garde donc son ombre. + let is_cutout = matches!(effect, Some((code, _)) if code == 1.0); + if cfg.shadow && !webcam_is_block && !is_cutout && g.shape_fade > 0.0 { self.draw_shadow( enc, g.w_dst, @@ -1629,6 +2218,35 @@ impl Compositor { WEBCAM_SHADOW_OPACITY * g.shape_fade, ); } + + // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra + // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement + // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de + // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et + // images y tombaient sur du noir, et le défaut EST une image. L'ordre est imposé : + // ombre, puis fond, puis caméra. + let (effect_code, blur_intensity) = match effect { + Some((code, e)) if code > 2.5 => { + self.draw_webcam_bg(enc, e.background.as_ref(), g.w_dst, g.w_px, g.w_radius); + (1.0, 0.0) + } + Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0)), + None => (0.0, 0.0), + }; + + // Metal tolère l'index 3 non lié tant que `fx.z` reste à 0 : la branche n'est + // pas prise, la texture n'est pas échantillonnée. Dès qu'il monte, elle doit + // l'être sur TOUT draw capable de la prendre — ici il n'y en a qu'un. L'état + // d'un encodeur est rémanent, donc lier avant le draw suffit, et l'ombre puis le + // fond qui précèdent sont en modes 1/2/5/6, que `ps_main` garde hors de la branche + // (`mode < 0.5`). + // + // Pas de déliaison après coup, contrairement au chemin Windows qui remet le slot + // t3 à `None` : cet état meurt avec l'encodeur, et les annotations en ouvrent un + // autre. Il n'y a rien sur quoi fuir. + if let Some(m) = mask.as_ref() { + enc.set_fragment_texture(3, Some(&m.tex)); + } self.draw_video( enc, &LayerCB { @@ -1637,10 +2255,13 @@ impl Compositor { quad_px: g.w_px, radius_px: g.w_radius, mode: 0.0, + // `color.a` porte l'alpha du découpage (`color.a * personne`) ; le RGB n'est + // plus lu, le fond ayant déjà été peint sous la caméra. color: [0.0, 0.0, 0.0, 1.0], + fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], src_prev: [u0, cv0, u1, cv1], dst_prev: g.w_dst_prev, - mb: [g.mb_taps, 1.0, 1.0, 0.0], + mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], ..Default::default() }, wy, @@ -1737,22 +2358,30 @@ impl Compositor { let y = cache.make_texture_from_pixel_buffer(out_tex, 0, metal::MTLPixelFormat::R8Unorm)?; let uv = cache.make_texture_from_pixel_buffer(out_tex, 1, metal::MTLPixelFormat::RG8Unorm)?; - let cmd_buf = self.gpu.context.new_command_buffer(); - for (target, pipeline) in [(&y, &self.pipeline_fs_y), (&uv, &self.pipeline_fs_uv)] { - let enc = self.begin_pass( - cmd_buf, - target, - Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), - pipeline, - )?; - enc.set_fragment_texture(0, Some(&self.rt)); - enc.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); - enc.end_encoding(); + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::Nv12Passes); + let cmd_buf = self.gpu.context.new_command_buffer(); + for (target, pipeline) in [(&y, &self.pipeline_fs_y), (&uv, &self.pipeline_fs_uv)] { + let enc = self.begin_pass( + cmd_buf, + target, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), + pipeline, + )?; + enc.set_fragment_texture(0, Some(&self.rt)); + enc.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + enc.end_encoding(); + } + self.submit(cmd_buf); } // Pas de miroir `Shared`, pas de `getBytes` : c'est tout l'intérêt. On attend // quand même, parce que `avcodec_send_frame` va lire ce buffer juste après. - self.submit(cmd_buf); - self.sync(); + // L'attente porte sur TOUT le travail GPU de la frame, composition comprise : + // `compose_frame` n'a fait que soumettre. + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::GpuWait); + self.sync(); + } Ok(()) } @@ -1935,7 +2564,718 @@ impl Compositor { #[cfg(test)] mod tests { - + use super::*; + + // ----------------------------------------------------------------------- + // Segmentation du sujet webcam + // + // Il n'y a PAS de banc hors Windows : `poc-d3d` est `cfg(windows)` dans son propre + // `Cargo.toml`, donc le `--cfg C8 --scene …` qui a prouvé le chemin Windows n'existe + // pas ici. Ce sont ces tests qui tiennent le rôle, et ils rendent de vrais pixels sur + // le device Metal du système plutôt que d'inspecter des champs : ce que le portage + // ajoute (une capture relue, un upload R8, une liaison à l'index 3, une branche + // `fx.z`) est précisément ce qu'aucun `cargo build` ne peut vérifier. + // ----------------------------------------------------------------------- + + /// Luma BT.709 limited d'un gris neutre : `yuv709_limited` fait `(Y - 16) / 219` sur + /// les trois canaux quand la chroma vaut 128, donc 235 rend du blanc franc et 16 du + /// noir franc. Ces deux valeurs rendent les assertions de couleur calculables à la main. + const Y_WHITE: u8 = 235; + const Y_BLACK: u8 = 16; + const UV_NEUTRAL: u8 = 128; + + fn region(w: u32, h: u32) -> metal::MTLRegion { + metal::MTLRegion { + origin: metal::MTLOrigin { x: 0, y: 0, z: 0 }, + size: metal::MTLSize { width: w as u64, height: h as u64, depth: 1 }, + } + } + + /// Une paire de plans NV12 synthétiques, sous forme de `MTLTexture` — ce que + /// `nv12_srvs` produirait d'une vraie frame, sans avoir à décoder quoi que ce soit. + fn nv12_textures( + device: &metal::Device, + w: u32, + h: u32, + luma: impl Fn(u32, u32) -> u8, + ) -> (metal::Texture, metal::Texture) { + let y = make_texture( + device, + metal::MTLPixelFormat::R8Unorm, + w, + h, + metal::MTLStorageMode::Shared, + metal::MTLTextureUsage::ShaderRead, + ); + let mut plane = vec![0u8; (w * h) as usize]; + for row in 0..h { + for col in 0..w { + plane[(row * w + col) as usize] = luma(col, row); + } + } + y.replace_region(region(w, h), 0, plane.as_ptr() as *const std::ffi::c_void, w as u64); + + let (uw, uh) = (w / 2, h / 2); + let uv = make_texture( + device, + metal::MTLPixelFormat::RG8Unorm, + uw, + uh, + metal::MTLStorageMode::Shared, + metal::MTLTextureUsage::ShaderRead, + ); + let chroma = vec![UV_NEUTRAL; (uw * uh * 2) as usize]; + uv.replace_region( + region(uw, uh), + 0, + chroma.as_ptr() as *const std::ffi::c_void, + (uw * 2) as u64, + ); + (y, uv) + } + + /// Masque 0 sur la moitié gauche, 255 sur la droite. La frontière tombe pile au milieu, + /// donc un échantillon pris au quart et un aux trois quarts sont loin du dégradé que le + /// filtrage linéaire pose sur la couture. + fn half_mask(w: u32, h: u32) -> Vec { + (0..w * h).map(|i| if i % w < w / 2 { 0u8 } else { 255u8 }).collect() + } + + #[test] + fn the_webcam_capture_comes_back_as_interleaved_rgb_at_model_resolution() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + // Moitié gauche noire, moitié droite blanche : la capture doit rendre les deux dans + // le bon sens. Une inversion d'axe passerait un test de taille sans se voir. + let (y, uv) = nv12_textures(&gpu.device, 64, 64, |col, _| { + if col < 32 { Y_BLACK } else { Y_WHITE } + }); + + let mut out = Vec::new(); + unsafe { + comp.capture_webcam_rgb( + &y, + &uv, + [0.0, 0.0, 1.0, 1.0], + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + &mut out, + ) + .expect("capture_webcam_rgb"); + } + + let (w, h) = ( + crate::segmentation::MODEL_WIDTH as usize, + crate::segmentation::MODEL_HEIGHT as usize, + ); + assert_eq!(out.len(), w * h * 3, "le modèle veut du RGB8 entrelacé, sans alpha"); + + let px = |buf: &[u8], col: usize, row: usize| -> [u8; 3] { + let i = (row * w + col) * 3; + [buf[i], buf[i + 1], buf[i + 2]] + }; + let left = px(&out, w / 4, h / 2); + let right = px(&out, 3 * w / 4, h / 2); + assert!(left.iter().all(|&c| c < 24), "moitié gauche pas noire : {left:?}"); + assert!(right.iter().all(|&c| c > 231), "moitié droite pas blanche : {right:?}"); + + // Deuxième capture sur le même buffer : c'est le régime établi (30 fois par + // seconde), et il ne doit ni réallouer ni traîner les octets du tour précédent. + let capacity = out.capacity(); + unsafe { + comp.capture_webcam_rgb( + &y, + &uv, + [0.0, 0.0, 1.0, 1.0], + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + &mut out, + ) + .expect("deuxième capture"); + } + assert_eq!(out.len(), w * h * 3); + assert_eq!(out.capacity(), capacity, "le scratch se réalloue d'une frame à l'autre"); + assert_eq!(px(&out, w / 4, h / 2), left); + assert_eq!(px(&out, 3 * w / 4, h / 2), right); + } + + #[test] + fn a_capture_of_zero_size_is_refused_rather_than_rendered() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let (y, uv) = nv12_textures(&gpu.device, 16, 16, |_, _| Y_WHITE); + let mut out = Vec::new(); + let err = unsafe { comp.capture_webcam_rgb(&y, &uv, [0.0, 0.0, 1.0, 1.0], 0, 144, &mut out) }; + assert!(err.is_err(), "une cible de largeur nulle doit être refusée"); + } + + #[test] + fn the_mask_texture_is_allocated_once_and_a_short_buffer_is_refused() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let (w, h) = (crate::segmentation::MODEL_WIDTH, crate::segmentation::MODEL_HEIGHT); + let mask = vec![255u8; (w * h) as usize]; + + comp.set_webcam_mask(&mask, w, h).expect("premier téléversement"); + let first = comp.webcam_mask.borrow().as_ref().map(|m| m.tex.as_ptr()); + comp.set_webcam_mask(&mask, w, h).expect("deuxième téléversement"); + let second = comp.webcam_mask.borrow().as_ref().map(|m| m.tex.as_ptr()); + assert_eq!( + first, second, + "la texture est recréée à chaque frame alors que la résolution du modèle est fixe" + ); + + // Un masque trop court doit être refusé, pas lu hors bornes : `replace_region` lit + // `width` octets par ligne sans rien savoir de la longueur de la tranche. + assert!(comp.set_webcam_mask(&mask[..(w * h) as usize - 1], w, h).is_err()); + assert!(comp.set_webcam_mask(&mask, 0, h).is_err()); + assert!(comp.clear_webcam_mask() == () && comp.webcam_mask.borrow().is_none()); + } + + /// Le test qui compte : le masque DÉCOUPE vraiment la caméra. + /// + /// Il rend le calque webcam plein cadre sur le RT avec `fx.z = 1` (détourage) et un + /// masque mi-fond mi-sujet, puis relit les pixels. Il couvre d'un coup les trois choses + /// que le portage ajoute et qu'aucune compilation ne vérifie : l'upload R8, la liaison + /// de la texture à l'index 3, et la branche `fx.z` de `ps_main` sur un vrai device. + #[test] + fn the_mask_actually_cuts_the_camera_out() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 64, 64).expect("Compositor::new_sized"); + comp.set_webcam_mask(&half_mask(8, 8), 8, 8).expect("set_webcam_mask"); + let (y, uv) = nv12_textures(&gpu.device, 16, 16, |_, _| Y_WHITE); + + // Fond bleu franc : une couleur que la caméra (blanche, chroma neutre) ne peut pas + // produire, donc « il reste du bleu » signifie « la caméra a été découpée ici ». + let cmd = gpu.context.new_command_buffer(); + let enc = comp + .begin_pass( + cmd, + &comp.rt, + Some(metal::MTLClearColor::new(0.0, 0.0, 1.0, 1.0)), + &comp.pipeline_main, + ) + .expect("begin_pass"); + { + let mask = comp.webcam_mask.borrow(); + enc.set_fragment_texture(3, Some(&mask.as_ref().expect("masque posé").tex)); + } + unsafe { + comp.draw_video( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src: [0.0, 0.0, 1.0, 1.0], + quad_px: [64.0, 64.0], + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + // fx.xy = étendue valide (toute la texture ici), fx.z = 1 → détourage. + fx: [1.0, 1.0, 1.0, 0.0], + src_prev: [0.0, 0.0, 1.0, 1.0], + dst_prev: [0.0, 0.0, 1.0, 1.0], + mb: [1.0, 1.0, 1.0, 0.0], + ..Default::default() + }, + &y, + &uv, + ); + } + enc.end_encoding(); + comp.submit(cmd); + let (rw, rh, rgba) = unsafe { comp.readback_direct().expect("readback_direct") }; + assert_eq!((rw, rh), (64, 64)); + + let px = |col: usize, row: usize| -> [u8; 4] { + let i = (row * rw as usize + col) * 4; + [rgba[i], rgba[i + 1], rgba[i + 2], rgba[i + 3]] + }; + let cut = px(16, 32); + let kept = px(48, 32); + assert_eq!(cut, [0, 0, 255, 255], "masque à 0 : le fond doit rester visible"); + assert_eq!(kept, [255, 255, 255, 255], "masque à 255 : la caméra doit rester opaque"); + } + + /// Même montage, mode fond personnalisé (`fx.z = 3`) : là où le masque dit « fond », le + /// shader doit peindre `color` — c'est le seul mode où `LayerCB::color` cesse d'être + /// du noir opaque décoratif et porte une valeur que le portage doit transmettre. + #[test] + fn the_custom_background_colour_replaces_the_masked_out_pixels() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 64, 64).expect("Compositor::new_sized"); + comp.set_webcam_mask(&half_mask(8, 8), 8, 8).expect("set_webcam_mask"); + let (y, uv) = nv12_textures(&gpu.device, 16, 16, |_, _| Y_WHITE); + + let cmd = gpu.context.new_command_buffer(); + let enc = comp + .begin_pass( + cmd, + &comp.rt, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), + &comp.pipeline_main, + ) + .expect("begin_pass"); + { + let mask = comp.webcam_mask.borrow(); + enc.set_fragment_texture(3, Some(&mask.as_ref().expect("masque posé").tex)); + } + unsafe { + comp.draw_video( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src: [0.0, 0.0, 1.0, 1.0], + quad_px: [64.0, 64.0], + mode: 0.0, + color: [1.0, 0.0, 0.0, 1.0], + fx: [1.0, 1.0, 3.0, 0.0], + src_prev: [0.0, 0.0, 1.0, 1.0], + dst_prev: [0.0, 0.0, 1.0, 1.0], + mb: [1.0, 1.0, 1.0, 0.0], + ..Default::default() + }, + &y, + &uv, + ); + } + enc.end_encoding(); + comp.submit(cmd); + let (rw, _, rgba) = unsafe { comp.readback_direct().expect("readback_direct") }; + let px = |col: usize, row: usize| -> [u8; 4] { + let i = (row * rw as usize + col) * 4; + [rgba[i], rgba[i + 1], rgba[i + 2], rgba[i + 3]] + }; + assert_eq!(px(16, 32), [255, 0, 0, 255], "fond masqué : la couleur custom doit peindre"); + assert_eq!(px(48, 32), [255, 255, 255, 255], "sujet : la caméra doit rester intacte"); + } + + + // ----------------------------------------------------------------------- + // `compose_frame` de bout en bout + // + // Les tests ci-dessus prouvent les pièces ; ceux-ci prouvent le CÂBLAGE — que + // `compose_frame` porte bien `fx`/`color` sur le calque webcam, qu'il lie le masque, et + // qu'il ne lève `fx.z` qu'une fois un masque réellement téléversé. Ils passent par de + // vraies `AVFrame` VideoToolbox (des `CVPixelBufferRef` IOSurface-backed), donc par le + // MÊME `nv12_srvs` que le décodeur : aucun raccourci n'est pris sur le seam de frame. + // + // Aucun n'a besoin d'ONNX Runtime : le masque est posé à la main par `set_webcam_mask`. + // C'est délibéré — ce que le portage ajoute côté GPU doit être vérifiable là où + // l'inférence n'est pas installée, ce qui est le cas de la CI. + // ----------------------------------------------------------------------- + + /// Une `AVFrame` VideoToolbox synthétique. `compose_frame` ne lit que `format`, + /// `data[3]`, `width` et `height` : le reste peut rester à zéro. + struct FakeFrame { + frame: Box, + _pb: crate::mac_frames::CVPixelBufferRef, + } + + impl FakeFrame { + fn new(w: u32, h: u32, luma: impl Fn(u32, u32) -> u8) -> FakeFrame { + let mut y = vec![0u8; (w * h) as usize]; + for row in 0..h { + for col in 0..w { + y[(row * w + col) as usize] = luma(col, row); + } + } + FakeFrame::from_planes(w, h, &y, &vec![UV_NEUTRAL; (w * (h / 2)) as usize]) + } + + fn from_planes(w: u32, h: u32, y: &[u8], uv: &[u8]) -> FakeFrame { + let pb = crate::mac_frames::nv12_pixel_buffer_from_planes(w, h, y, uv) + .expect("CVPixelBuffer NV12"); + let mut frame: Box = Box::new(unsafe { std::mem::zeroed() }); + frame.format = crate::ffi::AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX as i32; + frame.data[3] = pb.as_ptr() as *mut u8; + frame.width = w as i32; + frame.height = h as i32; + FakeFrame { frame, _pb: pb } + } + + fn as_ptr(&self) -> *const AVFrame { + &*self.frame as *const AVFrame + } + } + + /// Scène PiP minimale. `effect` est le JSON de `webcamEffect` (`"null"` pour aucun). + /// + /// `effects.shadow` vaut 0 À DESSEIN : ce curseur ne pilote plus que l'ombre de l'écran, + /// alors que celle du PiP est fixe (`WEBCAM_SHADOW_OPACITY`) et ne dépend que de + /// `cfg.shadow`. Le mettre à zéro est donc ce qui isole les deux — sinon un test sur + /// `cfg.shadow` mesure les deux ombres à la fois et ne dit plus rien de la caméra. + fn pip_scene_json(effect: &str) -> String { + format!( + r##"{{"clips":[], + "layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle", + "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects":{{"padding":0.18,"blur":false,"shadow":0,"roundnessFrac":0.05,"motionBlur":0}}, + "background":{{"kind":"color","color":"#0080ff"}}, + "zoomRegions":[],"annotations":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0, + "clipToBounds":false,"theme":"default"}}, + "cropByClip":[], + "webcamEffect":{effect}, + "output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + } + + /// Compose une frame et rend le RGBA du RT. `screen` est gris moyen, `webcam` blanche : + /// le blanc franc devient alors la SIGNATURE de la caméra, une couleur qu'aucun autre + /// calque de cette scène ne produit, donc comptable sans connaître la géométrie du PiP. + /// + /// Le fond est un bleu franc et NON du noir : le PiP par défaut tombe dans la marge, hors + /// de l'écran, et une ombre noire sur un fond noir ne se voit pas — le contrôle du test + /// d'ombre passerait alors pour une suppression réussie. + fn compose_pip(comp: &super::Compositor, effect: &str, shadow: bool) -> Vec { + let scene = crate::scene::Scene::from_json(&pip_scene_json(effect)).expect("scene json"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_has_webcam(true); + comp.set_scene(Some(scene)); + + let screen = FakeFrame::new(128, 128, |_, _| 126); + let webcam = FakeFrame::new(64, 64, |_, _| Y_WHITE); + let mut cfg = crate::config::Cfg::c8(); + cfg.bg_blur = false; + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.mblur_n = 1; + cfg.shadow = shadow; + unsafe { + comp.compose_frame(screen.as_ptr(), webcam.as_ptr(), 0.0, &cfg) + .expect("compose_frame"); + let (_, _, rgba) = comp.readback_direct().expect("readback_direct"); + rgba + } + } + + /// Pixels quasi blancs = pixels de caméra encore visibles. + fn camera_pixels(rgba: &[u8]) -> usize { + rgba.chunks_exact(4) + .filter(|px| px[0] > 240 && px[1] > 240 && px[2] > 240) + .count() + } + + const NO_EFFECT: &str = "null"; + const CUTOUT: &str = r#"{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":null}"#; + + /// Le piège que le brief nomme : un mode SANS masque ne doit rien changer. + /// + /// `effect_code` doit rester à 0 tant que rien n'a été segmenté, sinon le détourage rend + /// une webcam invisible sur les premières frames — le temps que l'inférence rende son + /// premier masque, c'est-à-dire à chaque ouverture de l'éditeur. L'assertion est + /// octet pour octet : « inchangé » ne souffre pas d'à-peu-près. + #[test] + fn a_mode_without_a_mask_composites_exactly_like_no_effect_at_all() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let plain = compose_pip(&comp, NO_EFFECT, true); + let requested = compose_pip(&comp, CUTOUT, true); + assert!( + comp.webcam_mask.borrow().is_none(), + "aucun masque n'a été téléversé : `modelPath` est absent, donc rien ne segmente" + ); + assert!(camera_pixels(&plain) > 200, "la caméra n'est pas à l'écran, le test ne prouve rien"); + assert_eq!(plain, requested, "un mode sans masque a changé des pixels"); + } + + /// Et une fois le masque là, le détourage doit VRAIMENT découper — dans la bonne + /// proportion. Le masque couvre la moitié de la caméra, donc la moitié de ses pixels + /// doit disparaître. Compter plutôt que d'échantillonner un point évite de coder en dur + /// la géométrie du PiP, qui appartient à `plan_frame` et non à ce portage. + #[test] + fn compose_frame_cuts_the_camera_out_once_a_mask_exists() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let whole = camera_pixels(&compose_pip(&comp, NO_EFFECT, true)); + assert!(whole > 200, "la caméra n'est pas à l'écran, le test ne prouve rien"); + + let (mw, mh) = (crate::segmentation::MODEL_WIDTH, crate::segmentation::MODEL_HEIGHT); + comp.set_webcam_mask(&half_mask(mw, mh), mw, mh).expect("set_webcam_mask"); + let cut = camera_pixels(&compose_pip(&comp, CUTOUT, true)); + + let expected = whole as f32 / 2.0; + assert!( + (cut as f32 - expected).abs() < expected * 0.15, + "détourage : {cut} pixels de caméra restants pour ~{expected:.0} attendus \ + (entier : {whole})" + ); + } + + /// L'ombre portée du PiP doit disparaître en détourage : une ombre projetée par un + /// rectangle devenu invisible se lit comme un artefact. Le test le prouve sans jamais + /// localiser l'ombre — en détourage, `cfg.shadow` ne doit plus rien changer du tout. + /// + /// Le contrôle est ce qui empêche l'assertion d'être vide : sans effet, `cfg.shadow` + /// DOIT changer des pixels, sinon la première moitié passerait aussi pour une scène où + /// aucune ombre n'a jamais été dessinée. + #[test] + fn the_pip_shadow_is_suppressed_in_cutout_mode() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + assert_ne!( + compose_pip(&comp, NO_EFFECT, true), + compose_pip(&comp, NO_EFFECT, false), + "contrôle : sans effet, l'ombre du PiP doit bel et bien se voir" + ); + + let (mw, mh) = (crate::segmentation::MODEL_WIDTH, crate::segmentation::MODEL_HEIGHT); + comp.set_webcam_mask(&half_mask(mw, mh), mw, mh).expect("set_webcam_mask"); + assert_eq!( + compose_pip(&comp, CUTOUT, true), + compose_pip(&comp, CUTOUT, false), + "en détourage, l'ombre est encore dessinée" + ); + } + + /// Le tour complet, celui qui a besoin d'ONNX Runtime : capture → inférence → masque → + /// composite, entraîné par `compose_frame` seul. Se saute proprement sans la + /// bibliothèque, ce que fait la CI — cf. `segmentation::runtime_available`. + #[test] + fn the_whole_loop_produces_a_mask_from_compose_frame_alone() { + if !crate::segmentation::runtime_available() { + eprintln!("ONNX Runtime absent (ORT_DYLIB_PATH) — test sauté"); + return; + } + let model = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx"); + if !model.is_file() { + eprintln!("modèle absent ({}) — test sauté", model.display()); + return; + } + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let effect = format!( + r#"{{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":{}}}"#, + serde_json::to_string(&model.to_string_lossy()).expect("chemin sérialisable") + ); + + // Le limiteur est à 30 Hz : une frame par tour ne suffirait pas, et l'inférence est + // asynchrone. On laisse au worker le temps de rendre un masque, sans jamais + // l'attendre dans le rendu — ce qui est précisément le contrat. + let mut uploaded = false; + for _ in 0..40 { + let _ = compose_pip(&comp, &effect, true); + if comp.webcam_mask.borrow().is_some() { + uploaded = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(40)); + } + assert!( + uploaded, + "aucun masque n'est remonté : la boucle capture → inférence → upload est rompue" + ); + assert!( + !*comp.seg_failed.borrow(), + "la segmentation s'est éteinte d'elle-même" + ); + } + + // ----------------------------------------------------------------------- + // Harnais visuel (opt-in) + // + // Les tests ci-dessus prouvent le mécanisme sur des images synthétiques, où le masque + // est posé à la main et donc trivialement juste. Ils ne peuvent rien dire de la QUALITÉ + // du masque que le modèle produit sur une vraie caméra — et « un masque qui composite » + // n'est pas la même affirmation que « un masque qui est correct ». + // + // `poc-d3d` étant `cfg(windows)`, il n'existe aucun banc ici pour trancher ça. Ceci en + // tient lieu : on lui donne une photo, il rend les quatre modes et écrit des PNG à + // regarder. Même forme d'opt-in que `tests/compose_linux.rs` (variable d'environnement + // + skip propre), et pour la même raison : ça rend sur GPU et ça lit un fichier que le + // dépôt ne porte pas. + // + // ``` + // ORT_DYLIB_PATH=/chemin/libonnxruntime.dylib \ + // OPENSCREEN_SEG_CAM=camera.png \ + // OPENSCREEN_SEG_VISUAL=target/seg \ + // cargo test -p openscreen-compositor --lib seg_visual -- --nocapture + // ``` + // ----------------------------------------------------------------------- + + /// RGB8 → NV12 BT.709 limited. Inverse EXACT de `yuv709_limited` dans `shaders.metal` : + /// une autre matrice ferait dériver les couleurs du rendu et on croirait à un bug du + /// compositeur là où il n'y aurait qu'une conversion d'entrée fausse. + #[allow(clippy::type_complexity)] + fn rgb_to_nv12(rgb: &[u8], w: u32, h: u32) -> (Vec, Vec) { + let luma = |i: usize| -> (f32, f32, f32, f32) { + let (r, g, b) = ( + rgb[i * 3] as f32 / 255.0, + rgb[i * 3 + 1] as f32 / 255.0, + rgb[i * 3 + 2] as f32 / 255.0, + ); + (r, g, b, 0.2126 * r + 0.7152 * g + 0.0722 * b) + }; + let mut y = vec![0u8; (w * h) as usize]; + for i in 0..(w * h) as usize { + let (_, _, _, yl) = luma(i); + y[i] = (16.0 + 219.0 * yl).round().clamp(0.0, 255.0) as u8; + } + // Chroma au plus proche voisin : l'échantillon en haut à gauche de chaque bloc 2x2. + // Un vrai filtre ne changerait rien à ce que ce harnais donne à voir. + let mut uv = vec![0u8; (w * (h / 2)) as usize]; + for row in 0..h / 2 { + for col in 0..w / 2 { + let (r, _, b, yl) = luma(((row * 2) * w + col * 2) as usize); + let cb = 128.0 + 224.0 * ((b - yl) / 1.8556); + let cr = 128.0 + 224.0 * ((r - yl) / 1.5748); + let o = (row * w + col * 2) as usize; + uv[o] = cb.round().clamp(0.0, 255.0) as u8; + uv[o + 1] = cr.round().clamp(0.0, 255.0) as u8; + } + } + (y, uv) + } + + fn frame_from_png(path: &std::path::Path) -> FakeFrame { + let img = image::open(path) + .unwrap_or_else(|e| panic!("{} : {e}", path.display())) + .to_rgb8(); + // NV12 veut des dimensions paires ; on rogne d'un pixel plutôt que de rééchantillonner. + let (w, h) = (img.width() & !1, img.height() & !1); + let src = img.as_raw(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for row in 0..h { + let (d, s) = ((row * w * 3) as usize, (row * img.width() * 3) as usize); + rgb[d..d + (w * 3) as usize].copy_from_slice(&src[s..s + (w * 3) as usize]); + } + let (y, uv) = rgb_to_nv12(&rgb, w, h); + FakeFrame::from_planes(w, h, &y, &uv) + } + + #[test] + fn seg_visual_renders_the_four_modes_from_a_real_photo() { + let (Ok(out_dir), Ok(cam)) = ( + std::env::var("OPENSCREEN_SEG_VISUAL"), + std::env::var("OPENSCREEN_SEG_CAM"), + ) else { + eprintln!("harnais visuel : OPENSCREEN_SEG_VISUAL + OPENSCREEN_SEG_CAM absents — sauté"); + return; + }; + if !crate::segmentation::runtime_available() { + eprintln!("ONNX Runtime absent (ORT_DYLIB_PATH) — sauté"); + return; + } + let model = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx"); + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — sauté"); + return; + }; + std::fs::create_dir_all(&out_dir).expect("dossier de sortie"); + + let (rw, rh) = (1280u32, 720u32); + let comp = super::Compositor::new_sized(&gpu, rw, rh).expect("Compositor::new_sized"); + let webcam = frame_from_png(std::path::Path::new(&cam)); + let screen = match std::env::var("OPENSCREEN_SEG_SCREEN") { + Ok(p) => frame_from_png(std::path::Path::new(&p)), + // Sans capture d'écran sous la main, un damier : il rend le détourage lisible, + // là où un aplat laisserait croire à un fond simplement peint. + Err(_) => FakeFrame::new(640, 360, |col, row| { + if (col / 40 + row / 40) % 2 == 0 { 180 } else { 60 } + }), + }; + let model_json = serde_json::to_string(&model.to_string_lossy()).expect("chemin"); + + let mut wrote = Vec::new(); + for (name, effect) in [ + ("00-none", "null".to_string()), + ("01-cutout", format!(r#"{{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":{model_json}}}"#)), + ("02-blur", format!(r#"{{"mode":"blur","blurIntensity":0.8,"background":null,"modelPath":{model_json}}}"#)), + ("03-custom", format!(r##"{{"mode":"custom","blurIntensity":0,"background":{{"kind":"color","color":"#ff2d95"}},"modelPath":{model_json}}}"##)), + ] { + // Le masque arrive de façon asynchrone : on tourne jusqu'à ce qu'il soit là, ce + // qui est aussi une vérification en soi — la boucle du rendu ne l'attend jamais. + let mut rgba = Vec::new(); + for _ in 0..60 { + rgba = compose_visual(&comp, &screen, &webcam, &effect); + if effect == "null" || comp.webcam_mask.borrow().is_some() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(30)); + } + let path = format!("{out_dir}/seg-{name}.png"); + image::RgbaImage::from_raw(rw, rh, rgba) + .expect("dimensions du readback") + .save(&path) + .unwrap_or_else(|e| panic!("écriture {path} : {e}")); + wrote.push(path); + } + for p in &wrote { + println!("wrote {p}"); + } + assert!( + comp.webcam_mask.borrow().is_some(), + "aucun masque n'a été produit : les trois modes d'effet sont sans objet" + ); + } + + /// Caméra plein cadre (`camera-fullscreen`… sans région : on force le rect via + /// `webcamRect`), pour que le masque occupe toute l'image et se juge à taille réelle. + fn compose_visual( + comp: &super::Compositor, + screen: &FakeFrame, + webcam: &FakeFrame, + effect: &str, + ) -> Vec { + let json = format!( + r##"{{"clips":[], + "layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle", + "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, + "webcamRect":{{"x":0.06,"y":0.10,"width":0.55,"height":0.72}}}}, + "effects":{{"padding":0.10,"blur":false,"shadow":1,"roundnessFrac":0.02,"motionBlur":0}}, + "background":{{"kind":"gradient","angleDeg":45,"stops":["#1b2a4a","#0b0f1a"]}}, + "zoomRegions":[],"annotations":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0, + "clipToBounds":false,"theme":"default"}}, + "cropByClip":[], + "webcamEffect":{effect}, + "output":{{"width":1280,"height":720,"fps":30}}}}"## + ); + let scene = crate::scene::Scene::from_json(&json).expect("scene json"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_has_webcam(true); + comp.set_scene(Some(scene)); + let mut cfg = crate::config::Cfg::c8(); + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.mblur_n = 1; + unsafe { + comp.compose_frame(screen.as_ptr(), webcam.as_ptr(), 0.0, &cfg) + .expect("compose_frame"); + let (_, _, rgba) = comp.readback_direct().expect("readback_direct"); + rgba + } + } /// Le pendant macOS de `compositor_windows`'s `every_shader_entry_point_compiles`. /// diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index 51c3a3a28..e6ed30ecf 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -31,6 +31,13 @@ use windows::Win32::Graphics::Direct3D::{ use windows::Win32::Graphics::Direct3D11::*; use windows::Win32::Graphics::Dxgi::Common::*; +/// Budget du cache de textures image (`img_cache`), en octets. +/// +/// Doit tenir le JEU ACTIF d'une frame — au pire un wallpaper d'écran ET un fond de caméra, que +/// rien n'empêche d'être deux 7680x7680 à 225 Mo pièce. Sous ce seuil l'éviction ne peut plus +/// rendre de mémoire sans toucher au jeu actif, ce qu'elle refuse de faire. 512 Mo borne la fuite +/// (1 774 Mo mesurés en parcourant les 18 wallpapers livrés) en laissant le jeu actif résident. +const IMG_CACHE_BUDGET_BYTES: u64 = 512 * 1024 * 1024; @@ -46,6 +53,28 @@ use windows::Win32::Graphics::Dxgi::Common::*; + +/// Cadence de l'inférence. Pas 60 : une silhouette ne bouge pas de façon perceptible en +/// 16 ms, et c'est le seul levier mesuré qui divise le coût par deux sans toucher au modèle. +const SEGMENTATION_HZ: u32 = 30; + +/// Cible RGBA + staging CPU pour l'extraction de la frame webcam qui alimente le modèle. +struct SegCapture { + rtv: ID3D11RenderTargetView, + rt: ID3D11Texture2D, + staging: ID3D11Texture2D, + width: u32, + height: u32, +} + +/// Texture du masque de segmentation, recréée seulement quand la résolution du modèle change. +struct WebcamMask { + tex: ID3D11Texture2D, + srv: ID3D11ShaderResourceView, + width: u32, + height: u32, +} + pub struct Compositor { dev: ID3D11Device, ctx: ID3D11DeviceContext, @@ -122,9 +151,23 @@ pub struct Compositor { /// de la source). Séparé de `img_cache` : les wallpapers sont des chemins disque, ces images /// des data URL de plusieurs Mo qu'on ne veut pas utiliser comme clés de hachage. ann_img_cache: RefCell>, - /// Cache des textures wallpaper image (clé = chemin absolu) : décodage/upload une seule - /// fois, puis réutilisées par frame. (SRV, largeur, hauteur). - img_cache: RefCell>, + /// Cache des textures wallpaper image (clé = chemin absolu) : décodé et uploadé une fois, + /// puis réutilisé par frame. (SRV, largeur, hauteur, tick d'usage). + /// + /// « Une fois » et non « une fois pour la session » : l'entrée est évinçable dès qu'elle + /// sort du jeu actif d'une frame, et un retour dessus la rechargera — cf. `cached_image`. + img_cache: RefCell>, + /// Compteur d'accès de `img_cache`, pour l'ordre LRU. Un compteur plutôt que l'index de + /// frame : une frame touche plusieurs entrées, et il faut pouvoir les ordonner entre elles. + img_tick: std::cell::Cell, + /// Valeur de `img_tick` au début de la frame en cours. Tout ce qui a été touché depuis + /// appartient au jeu actif et ne peut pas être évincé — voir `cached_image`. + img_frame_start: std::cell::Cell, + /// Masque de segmentation du sujet webcam, R8 à la résolution du modèle. Écrit par + /// `set_webcam_mask` depuis le thread d'inférence, lu au moment de dessiner la webcam. + /// `None` tant qu'aucune frame n'a été segmentée — l'effet reste alors éteint plutôt que + /// de rendre une webcam invisible en mode détourage. + webcam_mask: RefCell>, /// Dimensions du RENDER TARGET en pixels — la taille à laquelle `compose_frame` /// rastérise réellement, et donc le dénominateur de TOUTE conversion /// normalisé↔px de ce fichier. @@ -150,6 +193,26 @@ pub struct Compositor { /// de prévisualisation demandée (variable, contrairement au `staging` fixe à /// OUT_W×OUT_H). Recréée quand la taille change — voir `readback_resized`. live_readback_staging: RefCell>, + /// Cible + staging pour extraire la frame webcam à la résolution du modèle de + /// segmentation. Créée à la première capture, jamais redimensionnée : le modèle a une + /// entrée fixe. + seg_capture: RefCell>, + /// Worker d'inférence, absent tant que `enable_segmentation` n'a pas été appelé. + seg_worker: RefCell>, + /// Segmenteur tenu SUR LE THREAD DE RENDU, utilisé à la place du worker en mode + /// déterministe. Voir `set_segmentation_deterministic`. + seg_sync: RefCell>, + /// Export : cadence par frame et inférence synchrone, au lieu de l'horloge et du worker. + seg_deterministic: std::cell::Cell, + /// Boîte aux lettres du worker. Le masque est déposé depuis le thread d'inférence et + /// téléversé depuis le thread de rendu : aucun appel D3D ne traverse de thread, malgré + /// le device multithread-protected qui l'autoriserait. + seg_inbox: std::sync::Arc>>>, + seg_rate: RefCell, + /// Frame RGB réutilisée d'une capture à l'autre. + seg_scratch: RefCell>, + /// Le chargement du modèle a échoué : ne pas réessayer à chaque frame. + seg_failed: RefCell, /// Staging NV12 du readback d'ENCODAGE (backend CPU) — même motif de cache par taille /// que `live_readback_staging`, mais en NV12 et non en RGBA : l'encodeur logiciel veut /// les plans Y/UV, pas des pixels RGBA. Voir `read_nv12_scaled`. @@ -556,9 +619,20 @@ impl Compositor { text_cache: RefCell::new(HashMap::new()), ann_img_cache: RefCell::new(HashMap::new()), img_cache: RefCell::new(HashMap::new()), + img_tick: std::cell::Cell::new(0), + img_frame_start: std::cell::Cell::new(0), + webcam_mask: RefCell::new(None), render_size: Cell::new((out_w, out_h)), resize_target: RefCell::new(None), live_readback_staging: RefCell::new(None), + seg_capture: RefCell::new(None), + seg_worker: RefCell::new(None), + seg_sync: RefCell::new(None), + seg_deterministic: std::cell::Cell::new(false), + seg_inbox: std::sync::Arc::new(std::sync::Mutex::new(None)), + seg_rate: RefCell::new(crate::segmentation::RateLimiter::new(SEGMENTATION_HZ)), + seg_scratch: RefCell::new(Vec::new()), + seg_failed: RefCell::new(false), nv12_readback_staging: RefCell::new(None), }) } @@ -767,18 +841,70 @@ impl Compositor { /// Fond wallpaper image (cover-fit). `path` = chemin absolu (résolu côté app). Décodé et /// uploadé une fois (cache), puis échantillonné en mode 6. Err → l'appelant retombe sur une /// couleur plate. Le rect uv `src` recouvre toute la sortie en rognant le débordement. + /// Ouvre une frame du point de vue de `img_cache` : tout ce qui sera touché après cet appel + /// est le jeu actif, et devient inévinçable jusqu'à la frame suivante. + fn begin_image_frame(&self) { + // `+ 1` : la première entrée de cette frame recevra `img_tick + 1`, et la protection + // porte sur `tick >= img_frame_start`. Sans le décalage on protégerait aussi la + // DERNIÈRE entrée de la frame précédente, qui n'appartient plus au jeu actif — le + // résident pourrait alors dépasser le budget d'une texture entière. + self.img_frame_start.set(self.img_tick.get() + 1); + } + + /// Texture d'un fichier image, décodée une seule fois puis réutilisée. + /// + /// Le cache était NON BORNÉ, et c'est un vrai coût : les wallpapers livrés pèsent 23,7 Mo sur + /// disque mais 1 774 Mo une fois décodés en RGBA8 — `wallpaper8.jpg` fait 7680x7680, soit + /// 225 Mo à lui seul. Parcourir le sélecteur les chargeait tous et n'en libérait aucun. + /// + /// L'éviction est LRU sous un budget en octets, et ne touche jamais une texture que la frame + /// EN COURS a déjà servie : sans ça, un fond d'écran et un fond de caméra un peu gros se + /// chasseraient l'un l'autre à chaque frame, et un décodage coûte 129 ms contre les ~3,5 ms + /// d'une frame. Si le jeu actif dépasse à lui seul le budget, on dépasse le budget. + unsafe fn cached_image(&self, path: &str) -> Result<(ID3D11ShaderResourceView, u32, u32)> { + let tick = self.img_tick.get() + 1; + self.img_tick.set(tick); + // La recherche est isolée dans un `let` pour que l'emprunt immuable soit relâché AVANT le + // `borrow_mut()` (sinon double-emprunt RefCell → panic sur la 1re frame image). + let hit = self.img_cache.borrow().get(path).cloned(); + if let Some((srv, w, h, _)) = hit { + self.img_cache.borrow_mut().insert(path.to_string(), (srv.clone(), w, h, tick)); + return Ok((srv, w, h)); + } + let (srv, w, h) = self.load_image_srv(path)?; + let mut cache = self.img_cache.borrow_mut(); + cache.insert(path.to_string(), (srv.clone(), w, h, tick)); + // La politique vit dans `frame_geometry` : les trois backends la partagent, comme la + // géométrie, plutôt que d'entretenir trois copies qui finiraient par diverger. + let entries: Vec<(String, u64, u64)> = cache + .iter() + .map(|(k, e)| (k.clone(), e.1 as u64 * e.2 as u64 * 4, e.3)) + .collect(); + let protect_from = self.img_frame_start.get(); + for key in + crate::frame_geometry::lru_evictions(&entries, IMG_CACHE_BUDGET_BYTES, protect_from) + { + cache.remove(&key); + } + Ok((srv, w, h)) + } + unsafe fn draw_image_bg(&self, path: &str, output_aspect: f32) -> Result<()> { - // NB : la recherche est isolée dans un `let` pour que l'emprunt immuable soit relâché - // AVANT le `borrow_mut()` (sinon double-emprunt RefCell → panic sur la 1re frame image). - let cached = self.img_cache.borrow().get(path).cloned(); - let (srv, iw, ih) = match cached { - Some(v) => v, - None => { - let loaded = self.load_image_srv(path)?; - self.img_cache.borrow_mut().insert(path.to_string(), loaded.clone()); - loaded - } - }; + self.draw_image_in(path, [0.0, 0.0, 1.0, 1.0], [0.0, 0.0], 0.0, output_aspect) + } + + /// `draw_image_bg` pour un rect quelconque — la bulle webcam s'en sert avec ses coins + /// arrondis. `output_aspect` est le ratio du RECT visé, pas celui de la sortie : le crop + /// « cover » se calcule contre la zone qu'on remplit. + unsafe fn draw_image_in( + &self, + path: &str, + dst: [f32; 4], + quad_px: [f32; 2], + radius_px: f32, + output_aspect: f32, + ) -> Result<()> { + let (srv, iw, ih) = self.cached_image(path)?; let ai = iw as f32 / ih as f32; // Le fond remplit TOUJOURS le cadre (dst=[0,0,1,1], jamais rétréci par `undistort`), // mais le canvas interne est un 16:9 fixe étiré ensuite vers le VRAI ratio de sortie @@ -795,8 +921,10 @@ impl Compositor { (0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5) }; self.upload_cb(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], + dst, src: [u0, v0, u1, v1], + quad_px, + radius_px, mode: 6.0, ..Default::default() }); @@ -805,6 +933,70 @@ impl Compositor { Ok(()) } + /// Peint le fond du mode « personnalisé » DANS la bulle webcam, avant que la caméra n'y soit + /// découpée par-dessus. + /// + /// Le shader ne sait peindre qu'une couleur plate sous le masque, donc un dégradé ou une image + /// y tombaient sur du noir — et le défaut EST une image (`DEFAULT_WALLPAPER`), si bien que le + /// mode ne rendait jamais ce que le sélecteur montrait. Peindre le fond puis composer la + /// caméra en détourage donne exactement le même résultat (`lerp(fond, caméra, personne)`, ici + /// par le mélange alpha) pour les trois sortes de fond, en réutilisant les chemins déjà + /// éprouvés du fond d'écran, et sans rien ajouter aux trois shaders. + /// + /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit épouser ses coins arrondis, + /// sinon un rectangle déborde derrière la caméra. + unsafe fn draw_webcam_bg( + &self, + bg: Option<&SceneBackground>, + dst: [f32; 4], + quad_px: [f32; 2], + radius_px: f32, + ) { + const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + let solid = |color: [f32; 4]| LayerCB { + dst, + quad_px, + radius_px, + mode: 1.0, + color, + ..Default::default() + }; + match bg { + Some(SceneBackground::Color { color }) => { + self.draw_solid(&solid(parse_hex(color).unwrap_or(BLACK))); + } + Some(SceneBackground::Gradient { angle_deg, stops }) => { + let c0 = stops.first().and_then(|s| parse_hex(s)).unwrap_or(BLACK); + let c1 = stops.last().and_then(|s| parse_hex(s)).unwrap_or(c0); + // angle CSS → direction unitaire, même convention que le fond d'écran. + let a = angle_deg.to_radians(); + let dir = [a.sin(), -a.cos()]; + self.draw_solid(&LayerCB { + dst, + quad_px, + radius_px, + src: [c1[0], c1[1], c1[2], c1[3]], + mode: 5.0, + color: c0, + fx: [dir[0], dir[1], 0.0, 0.0], + ..Default::default() + }); + } + Some(SceneBackground::Image { path }) => { + // Même contrat que le fond d'écran : un chemin cassé est loggé puis remplacé par + // du noir. Un fallback silencieux redonnerait le bug qu'on corrige. + let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; + if let Err(e) = self.draw_image_in(path, dst, quad_px, radius_px, aspect) { + eprintln!("[compositor] fond webcam \"{}\" : {:#}", path, e); + self.draw_solid(&solid(BLACK)); + } + } + // Personnalisé sans fond : noir, comme avant — mais c'est désormais le seul chemin + // qui y mène, au lieu de l'être pour toute image et tout dégradé. + None => self.draw_solid(&solid(BLACK)), + } + } + /// Décode un fichier image (jpg/png) → texture RGBA immuable + SRV. unsafe fn load_image_srv(&self, path: &str) -> Result<(ID3D11ShaderResourceView, u32, u32)> { // Les annotations image stockent une data URL (cf. `types.ts` : « Separate storage for @@ -846,6 +1038,341 @@ impl Compositor { Ok((srv.unwrap(), w, h)) } + /// Extrait la frame webcam en RGB8 à la résolution du modèle, dans `out`. + /// + /// `src` est le rect source de la webcam en UV (le même que celui passé à `draw_video`), + /// donc le crop utilisateur et le miroir sont déjà dedans — le modèle voit exactement ce + /// que le spectateur verra, et le masque n'a pas à être recadré après coup. + /// + /// **À appeler AVANT `begin()`** : la méthode réquisitionne la cible de rendu et le + /// viewport, et ne les restaure pas. Les appeler dans l'autre ordre dessinerait la scène + /// dans une texture de 256x144. + /// + /// C'est le seul readback GPU->CPU du chemin. Il porte 256x144x4 = 147 Ko, contre la + /// frame entière que la preview lit déjà à chaque image ; sur le chemin export, qui lui + /// est GPU-résident de bout en bout, c'est en revanche un point de synchronisation neuf + /// et c'est là qu'il faudra le mesurer. + pub unsafe fn capture_webcam_rgb( + &self, + wy: &ID3D11ShaderResourceView, + wuv: &ID3D11ShaderResourceView, + src: [f32; 4], + width: u32, + height: u32, + out: &mut Vec, + ) -> Result<()> { + if width == 0 || height == 0 { + bail!("capture webcam de dimensions nulles ({width}x{height})"); + } + { + let mut slot = self.seg_capture.borrow_mut(); + if !matches!(slot.as_ref(), Some(c) if c.width == width && c.height == height) { + let td = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_R8G8B8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_RENDER_TARGET.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut rt: Option = None; + self.dev.CreateTexture2D(&td, None, Some(&mut rt))?; + let rt = rt.unwrap(); + let mut rtv: Option = None; + self.dev.CreateRenderTargetView(&rt, None, Some(&mut rtv))?; + + let sd = D3D11_TEXTURE2D_DESC { + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, + ..td + }; + let mut staging: Option = None; + self.dev.CreateTexture2D(&sd, None, Some(&mut staging))?; + + *slot = Some(SegCapture { + rtv: rtv.unwrap(), + rt, + staging: staging.unwrap(), + width, + height, + }); + } + } + + let cap = self.seg_capture.borrow(); + let cap = cap.as_ref().expect("créé juste au-dessus"); + + self.bind_compose_state(); + self.ctx.OMSetBlendState(&self.blend_none, None, 0xffffffff); + self.ctx.OMSetRenderTargets(Some(&[Some(cap.rtv.clone())]), None); + let vp = D3D11_VIEWPORT { + TopLeftX: 0.0, TopLeftY: 0.0, + Width: width as f32, Height: height as f32, MinDepth: 0.0, MaxDepth: 1.0, + }; + self.ctx.RSSetViewports(Some(&[vp])); + // Plein cadre de la cible, sans coins ni motion blur : le modèle veut l'image, pas + // la mise en forme. + self.draw_video( + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src, + quad_px: [width as f32, height as f32], + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + mb: [1.0, 1.0, 1.0, 0.0], + ..Default::default() + }, + wy, + wuv, + ); + + self.ctx.CopyResource(&cap.staging, &cap.rt); + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + self.ctx.Map(&cap.staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped))?; + out.clear(); + out.reserve((width * height * 3) as usize); + for row in 0..height as usize { + let line = (mapped.pData as *const u8).add(row * mapped.RowPitch as usize); + for col in 0..width as usize { + let px = line.add(col * 4); + // RGBA -> RGB : le modèle n'a pas de canal alpha en entrée. + out.push(*px); + out.push(*px.add(1)); + out.push(*px.add(2)); + } + } + self.ctx.Unmap(&cap.staging, 0); + Ok(()) + } + + /// Publie le masque de segmentation du sujet webcam (R8, `width`x`height`, 0 = fond). + /// + /// Appelé depuis le thread d'inférence, pas depuis le thread de rendu — d'où le + /// `SetMultithreadProtected(true)` posé à la création du device (`d3d_windows.rs`). La + /// texture est `DYNAMIC` et réécrite en place ; elle n'est recréée que si la résolution du + /// modèle change, ce qui n'arrive pas en régime établi. + pub fn set_webcam_mask(&self, data: &[u8], width: u32, height: u32) -> Result<()> { + if width == 0 || height == 0 { + bail!("masque webcam de dimensions nulles ({width}x{height})"); + } + let expected = (width as usize) * (height as usize); + if data.len() < expected { + bail!("masque webcam trop court : {} octets pour {width}x{height}", data.len()); + } + + let mut slot = self.webcam_mask.borrow_mut(); + let needs_alloc = !matches!(slot.as_ref(), Some(m) if m.width == width && m.height == height); + if needs_alloc { + let td = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_R8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DYNAMIC, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32, + MiscFlags: 0, + }; + let mut tex: Option = None; + unsafe { self.dev.CreateTexture2D(&td, None, Some(&mut tex))? }; + let tex = tex.unwrap(); + let mut srv: Option = None; + unsafe { self.dev.CreateShaderResourceView(&tex, None, Some(&mut srv))? }; + *slot = Some(WebcamMask { tex, srv: srv.unwrap(), width, height }); + } + + let mask = slot.as_ref().expect("alloué juste au-dessus"); + unsafe { + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + self.ctx.Map(&mask.tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut mapped))?; + // `RowPitch` n'est pas `width` : le driver aligne les lignes, donc on recopie + // ligne à ligne plutôt que d'un bloc. + for row in 0..height as usize { + let dst = (mapped.pData as *mut u8).add(row * mapped.RowPitch as usize); + let src = data.as_ptr().add(row * width as usize); + std::ptr::copy_nonoverlapping(src, dst, width as usize); + } + self.ctx.Unmap(&mask.tex, 0); + } + Ok(()) + } + + /// Un tour de segmentation : téléverse le masque prêt, puis soumet une nouvelle frame si + /// la cadence l'autorise. + /// + /// Les deux moitiés sont volontairement désynchronisées. Le masque téléversé ici vient de + /// la frame précédente — une frame de retard sur une silhouette est invisible, alors + /// qu'attendre l'inférence bloquerait le rendu, ce qui est exactement le coût que toute + /// cette conception cherche à ne pas payer. + unsafe fn pump_segmentation( + &self, + wy: &ID3D11ShaderResourceView, + wuv: &ID3D11ShaderResourceView, + valid: [f32; 2], + ) -> Result<()> { + if *self.seg_failed.borrow() { + return Ok(()); + } + // Rien à faire si aucun effet n'est demandé : ni capture, ni inférence, ni masque. + // Le coût de la fonctionnalité est alors exactement nul. + let (wants_effect, model_path) = { + let scene = self.scene.borrow(); + match scene.as_ref().and_then(|s| s.webcam_effect.as_ref()) { + Some(e) if e.shader_code() > 0.0 => (true, e.model_path.clone()), + _ => (false, None), + } + }; + if !wants_effect { + return Ok(()); + } + + // Démarrage paresseux, piloté par la scène : personne n'a à appeler + // `enable_segmentation` à la main, et un modèle introuvable éteint l'effet au lieu + // de faire tomber le rendu. + if self.seg_worker.borrow().is_none() && self.seg_sync.borrow().is_none() { + let Some(path) = model_path else { return Ok(()) }; + if let Err(e) = self.enable_segmentation(std::path::Path::new(&path)) { + eprintln!("[segmentation] désactivée : {e}"); + // Une scène qui reste identique retenterait à chaque frame ; on pose un + // worker vide plutôt que de journaliser 60 fois par seconde. + *self.seg_failed.borrow_mut() = true; + return Ok(()); + } + // En preview on rend cette frame sans masque : le worker vient de démarrer et + // l'effet apparaîtra dans quelques millisecondes, ce que personne ne voit. À + // l'export cette frame part dans le fichier — on enchaîne donc sur la capture et + // l'inférence plutôt que de la laisser sortir non détourée. + if !self.seg_deterministic.get() { + return Ok(()); + } + } + + if let Some(mask) = self.seg_inbox.lock().unwrap().take() { + self.set_webcam_mask( + &mask, + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + )?; + } + + // La cadence horloge est le bon réglage en preview et le mauvais à l'export, où les + // frames défilent aussi vite que la machine décode : le nombre de frames couvertes par + // un masque dépendrait alors de la charge. En déterministe, une inférence par frame. + if !self.seg_deterministic.get() + && !self.seg_rate.borrow_mut().should_run(std::time::Instant::now()) + { + return Ok(()); + } + let mut scratch = self.seg_scratch.borrow_mut(); + // La frame ENTIÈRE, pas le sous-rect dessiné : un crop utilisateur serré amputerait + // le sujet en entrée du modèle, et le masque serait faux là où il compte le plus. + // Le shader ramène ses coordonnées dans cet espace via `fx.xy`. + self.capture_webcam_rgb( + wy, + wuv, + [0.0, 0.0, valid[0], valid[1]], + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + &mut scratch, + )?; + if self.seg_deterministic.get() { + // Synchrone : le masque doit exister avant que cette frame ne soit composée, sinon + // on retombe sur le défaut qu'on corrige. Une inférence ratée laisse le masque + // précédent, comme le fait le worker. + let mut sync = self.seg_sync.borrow_mut(); + if let Some(seg) = sync.as_mut() { + match seg.run(&scratch) { + Ok(mask) => { + let mask = mask.to_vec(); + drop(sync); + self.set_webcam_mask( + &mask, + crate::segmentation::MODEL_WIDTH, + crate::segmentation::MODEL_HEIGHT, + )?; + } + Err(e) => eprintln!("[segmentation] frame ignorée : {e}"), + } + } + } else if let Some(w) = self.seg_worker.borrow().as_ref() { + w.submit(&scratch); + } + Ok(()) + } + + /// Démarre la segmentation du sujet webcam pour ce compositeur. + /// + /// Idempotent. Tant qu'elle n'est pas appelée, `compose_frame` ne fait rien de plus et + /// la webcam se dessine comme avant — c'est ce qui rend l'effet inerte plutôt que cassé + /// sur une build sans modèle. + pub fn enable_segmentation(&self, model_path: &std::path::Path) -> Result<()> { + if self.seg_worker.borrow().is_some() || self.seg_sync.borrow().is_some() { + return Ok(()); + } + let segmenter = crate::segmentation::Segmenter::load(model_path)?; + // En déterministe, le segmenteur reste ici : l'inférence tourne sur le thread de rendu, + // donc le masque de la frame N est prêt AVANT qu'elle ne soit composée. Le worker est un + // choix de preview — ne jamais bloquer l'affichage — et c'est exactement ce qui rend + // l'export irreproductible, le masque arrivant quelques frames plus tard selon la charge. + if self.seg_deterministic.get() { + *self.seg_sync.borrow_mut() = Some(segmenter); + return Ok(()); + } + let inbox = std::sync::Arc::clone(&self.seg_inbox); + let worker = crate::segmentation::SegmentationWorker::spawn(segmenter, move |mask, _, _| { + // Écrase le masque précédent s'il n'a pas encore été téléversé : c'est le plus + // récent qui vaut, jamais une file. + *inbox.lock().unwrap() = Some(mask.to_vec()); + }); + *self.seg_worker.borrow_mut() = Some(worker); + Ok(()) + } + + /// Bascule la segmentation en mode reproductible, pour l'export. + /// + /// En preview, la cadence suit l'horloge (30 Hz réels) et l'inférence tourne sur un worker : + /// c'est le bon choix, l'affichage ne doit jamais attendre. À l'export les frames sont rendues + /// aussi vite que la machine décode, sans rapport avec le temps réel — et ces deux choix + /// deviennent alors des bugs. La cadence horloge fait dépendre le nombre de frames couvertes + /// par un masque de la vitesse de la machine, et le worker asynchrone rend les premières + /// frames AVANT que le premier masque n'existe : elles partent dans le fichier avec le vrai + /// arrière-plan de la webcam. Deux exports du même projet ne donnent donc pas les mêmes + /// pixels, ce qui casse l'invariant « l'export est identique à la preview ». + /// + /// En déterministe : une inférence PAR FRAME, synchrone. Plus coûteux (~3 ms/frame), mais + /// l'export est hors ligne et chaque frame porte le masque calculé depuis SA propre image. + /// + /// À appeler avant la première frame — c'est ce qui décide comment `enable_segmentation` + /// s'installe. + pub fn set_segmentation_deterministic(&self, on: bool) { + if self.seg_deterministic.get() == on { + return; + } + self.seg_deterministic.set(on); + // Changer de mode change le MOTEUR, et `enable_segmentation` est idempotent sur la + // PRÉSENCE d'un moteur : sans démonter celui qui ne correspond plus, le drapeau mentirait. + // Un compositeur qui a déjà servi en preview garderait son worker, `seg_sync` resterait + // vide, et l'export entier ne ferait AUCUNE inférence. Le démarrage paresseux de + // `pump_segmentation` réinstalle le bon moteur à la frame suivante. + *self.seg_worker.borrow_mut() = None; + *self.seg_sync.borrow_mut() = None; + // Et le masque que le worker démonté avait peut-être déjà déposé : il vient de l'autre + // mode, il n'a rien à faire sur la première frame de celui-ci. + *self.seg_inbox.lock().unwrap() = None; + } + + /// Éteint l'effet : la webcam se redessine telle quelle à la frame suivante. + pub fn clear_webcam_mask(&self) { + *self.webcam_mask.borrow_mut() = None; + } + pub fn set_cursor(&self, track: CursorTrack) { *self.cursor.borrow_mut() = Some(track); } @@ -899,15 +1426,7 @@ impl Compositor { clip: [f32; 4], ) -> Result<()> { let path = sprite.path.as_str(); - let cached = self.img_cache.borrow().get(path).cloned(); - let (srv, iw, ih) = match cached { - Some(v) => v, - None => { - let loaded = self.load_image_srv(path)?; - self.img_cache.borrow_mut().insert(path.to_string(), loaded.clone()); - loaded - } - }; + let (srv, iw, ih) = self.cached_image(path)?; let ar = iw as f32 / ih as f32; let (pw, ph) = if ar >= 1.0 { (size_px, size_px / ar) } else { (size_px * ar, size_px) }; let hotspot = [sprite.hotspot_x, sprite.hotspot_y]; @@ -1091,26 +1610,26 @@ impl Compositor { frame: f32, cfg: &Cfg, ) -> Result<()> { + self.begin_image_frame(); let (sy, suv) = self.nv12_srvs(screen)?; let (wy, wuv) = self.nv12_srvs(webcam)?; let (stw, sth) = self.tex_dims(screen); let (wtw, wth) = self.tex_dims(webcam); let (scw, sch) = ((*screen).width as f32, (*screen).height as f32); let (wcw, wch) = ((*webcam).width as f32, (*webcam).height as f32); + // Étendue valide de la texture webcam : les décodeurs allouent des textures alignées, + // donc la frame n'occupe pas forcément toute la texture. + let w_valid = [wcw / wtw as f32, wch / wth as f32]; + + // Segmentation, AVANT `begin()` : la capture réquisitionne la cible de rendu. + self.pump_segmentation(&wy, &wuv, w_valid)?; let u_max = scw / stw as f32; let v_max = sch / sth as f32; let scene_ref = self.scene.borrow(); let cursor_ref = self.cursor.borrow(); let lp = *self.live_params.borrow(); - // Toute la géométrie vit dans `frame_geometry::plan_frame` — 353 lignes sans un - // appel GPU, partagées avec le backend Metal. Ce qui suit ce destructure est - // inchangé, à l'octet près. - let crate::frame_geometry::FrameGeometry { - scene_preset, mb_taps, source_t, zoom_rotation, padding_scale, cut, s_dst, - s_dst_prev, s_ann, s_radius, frame_min_px, w_dst, w_dst_prev, w_px, w_radius, - shape_fade, - } = crate::frame_geometry::plan_frame(&crate::frame_geometry::FrameGeometryInput { + let g = crate::frame_geometry::plan_frame(&crate::frame_geometry::FrameGeometryInput { render_px: [self.rw(), self.rh()], screen_tex_px: [stw as f32, sth as f32], screen_visible_px: [scw, sch], @@ -1124,6 +1643,23 @@ impl Compositor { cursor: cursor_ref.as_ref(), timeline_t_override: *self.timeline_t_override.borrow(), }); + let scene_preset = g.scene_preset.clone(); + let mb_taps = g.mb_taps; + let mb_amount = g.mb_amount; + let source_t = g.source_t; + let zoom_rotation = g.zoom_rotation; + let _padding_scale = g.padding_scale; + let cut = g.cut; + let s_dst = g.s_dst; + let s_dst_prev = g.s_dst_prev; + let s_ann = g.s_ann; + let s_radius = g.s_radius; + let frame_min_px = g.frame_min_px; + let w_dst = g.w_dst; + let w_dst_prev = g.w_dst_prev; + let w_px = g.w_px; + let w_radius = g.w_radius; + let shape_fade = g.shape_fade; self.begin([0.0, 0.0, 0.0, 1.0]); @@ -1268,7 +1804,7 @@ impl Compositor { color: [0.0, 0.0, 0.0, 1.0], src_prev: [su0_p, sv0_p, su0_p + 2.0 * hu_p, sv0_p + 2.0 * hv_p], dst_prev: s_dst_prev, - mb: [mb_taps, 1.0, 1.0, 0.0], + mb: [mb_taps, mb_amount, 1.0, 0.0], ..Default::default() }, &sy, @@ -1330,175 +1866,71 @@ impl Compositor { } // --- curseur custom : suit le mapping src/dst (zoom+layout), click bounce, - // et flou de mouvement par fantômes le long de sa vélocité (frame-1 -> frame) --- - // Jeu de sprites résolu par l'app (art du thème + art intégrée pour les états qu'il ne - // fournit pas), sinon math dot+ring — fixture/bench sans scène uniquement. - let cursor_sprites: HashMap = self - .scene - .borrow() - .as_ref() - .map(|s| s.cursor.cursor_sprites.clone()) - .unwrap_or_default(); - // « Clip to canvas » : tronque le curseur aux bords de l'écran (utile quand le padding - // crée une marge et que la pointe, près du bord de la vidéo, dépasserait dedans). - // Rect englobant tout par défaut = pas d'effet (le mode 4/7 du shader clippe sur `fx`). - // Écran incliné : le rect droit d'origine rognerait le curseur sur les parties du plan - // qui débordent au-dessus/en dessous, donc on clippe sur la bbox du quad projeté. Un - // rect reste une approximation du quadrilatère — `fx` ne sait pas exprimer autre chose — - // mais qui ne coupe plus rien de ce qui est réellement affiché. - let cursor_bounds: [f32; 4] = match tilt.as_ref() { - None => s_dst, - Some(quad) => { - let (hx, hy) = quad.half_extents_px(); - [ - (quad_center_px[0] - hx) / self.rw(), - (quad_center_px[1] - hy) / self.rh(), - 2.0 * hx / self.rw(), - 2.0 * hy / self.rh(), - ] - } - }; - let cursor_clip_rect: [f32; 4] = match self.scene.borrow().as_ref() { - Some(s) if s.cursor.clip_to_bounds => cursor_bounds, - _ => [-1.0, -1.0, 3.0, 3.0], - }; - // « Show cursor » : piloté par la scène (contrat de l'app) quand elle est posée ; sinon - // par `cfg.cursor` (inspector / bench fixture). - let cursor_show = scene_ref - .as_ref() - .map(|s| s.cursor.show) - .unwrap_or(cfg.cursor); - if cursor_show { - let cursor_ref = self.cursor.borrow(); - if let Some(track) = cursor_ref.as_ref() { - let t = self.cursor_t_override.borrow().unwrap_or(frame / FPS); - // position sortie à un temps donné via un mapping screen (src rect + dst) - let map = |cxy: Option<(f32, f32)>, s0: [f32; 2], h: [f32; 2], dst: [f32; 4]| { - cxy.and_then(|(cx2, cy2)| { - let fx = (cx2 * u_max - s0[0]) / (2.0 * h[0]); - let fy = (cy2 * v_max - s0[1]) / (2.0 * h[1]); - if !(0.0..=1.0).contains(&fx) || !(0.0..=1.0).contains(&fy) { - return None; - } - // Écran incliné : le curseur vit SUR le plan, pas dans un calque - // au-dessus. On garde donc sa position dans le repère DU PLAN et c'est - // le dessin qui projette — position ET sprite. Le poser sur `dst`, le - // rect droit d'origine, le laissait flotter à côté de l'image, l'écart - // se comptant en dizaines de pixels là où le plan s'éloigne le plus. - Some(match tilt.as_ref() { - Some(&quad) => CursorPlacement::Tilted { - plane_pt: [fx, fy], - quad, - center_px: quad_center_px, - screen_px: s_px, - render_px: [self.rw(), self.rh()], - }, - None => CursorPlacement::Upright { - center: [dst[0] + fx * dst[2], dst[1] + fy * dst[3]], - }, - }) - }) - }; - let raw_xy = track.at(t); - // Hors de [0,1] = pointeur hors du rect source actuel (zoom serré / hors écran) — - // état normal en cours de lecture, pas une erreur : rien à dessiner cette frame. - let mapped = map(raw_xy, [su0, sv0], [hu, hv], s_dst); - if let Some(cur) = mapped { - // taille + amplitude du bounce pilotées par l'inspector (défauts = fixture). - // `padding_scale` : le curseur est un recouvrement synthétique, pas cuit dans - // la vidéo — quand le padding rétrécit l'écran, le curseur doit rétrécir - // pareil pour rester à l'échelle du contenu (sinon sa pointe semble se - // décaler/dériver à mesure que le padding grandit). - let bounce = 1.0 + (track.bounce(t) - 1.0) * lp.cursor_bounce_scale; - // Pas de facteur de tilt ici : sur un plan incliné la taille est convertie - // en fraction du plan puis projetée avec lui (voir `draw_cursor_sprite`), - // donc la réduction vient de la projection. L'ajouter en plus rétrécirait - // le curseur deux fois. - let sz = CURSOR_BASE_SIZE_FRAC - * frame_min_px - * lp.cursor_size_scale - * bounce - * padding_scale; - // flou de mouvement DU CURSEUR, indépendant de cfg.mblur_n (écran/vidéo). - // BUG corrigé : augmenter l'intensité ne faisait auparavant que sur-échantillonner - // (plus de taps) un écart figé d'1 frame (1/60s) -> la traînée ne s'allongeait - // JAMAIS, donc restait quasi invisible quel que soit le réglage. L'intensité doit - // étirer la FENÊTRE temporelle de la traînée, pas seulement sa densité d'échantillons. - // 0 -> 1 frame en arrière (net) ; 1 -> ~8 frames (~130 ms à 60fps, traînée nette). - let blur01 = lp.cursor_motion_blur.clamp(0.0, 1.0); - let has_scene = self.scene.borrow().is_some(); - let trail_frames = if has_scene { 1.0 + blur01 * 7.0 } else { 1.0 }; - // BUG corrigé : le plancher était 2 (pas 1) -> même à blur=0 le curseur - // passait TOUJOURS par le chemin additif multi-tap (poids 1/taps=0.5 chacun), - // et comme prev≠cur au pixel près, les deux copies à 0.5 d'alpha ne se - // recouvraient jamais parfaitement -> curseur en permanence semi-transparent - // (quasi invisible sur fond clair), même sans aucun flou demandé. - let taps = if has_scene { - (1.0 + blur01 * 10.0).round() as u32 // 0 -> 1 (net) ; 1 -> 11 (traînée) - } else { - cfg.mblur_n // fixture/bench : comportement historique inchangé - }; - // L'état est celui de l'instant rendu : la traînée de flou reprend le même - // sprite pour toutes ses copies, un changement d'état en plein mouvement - // n'a pas à laisser une traînée hybride. - let cursor_type = track.type_at(t).map(str::to_string); - let cursor_type = cursor_type.as_deref(); - if taps <= 1 { + // et flou de mouvement (parité `compositor_macos.rs` et `compositor_linux.rs`) --- + if let Some(track) = cursor_ref.as_ref() { + let plan = crate::frame_geometry::plan_cursor( + &g, + &crate::frame_geometry::CursorPlanInput { + render_px: [self.rw(), self.rh()], + u_max, + v_max, + cfg, + live: lp, + scene: scene_ref.as_ref(), + track, + t: self.cursor_t_override.borrow().unwrap_or(frame / FPS), + }, + ); + if let Some(plan) = plan { + let cursor_sprites: HashMap = scene_ref + .as_ref() + .map(|s| s.cursor.cursor_sprites.clone()) + .unwrap_or_default(); + let cursor_type = plan.cursor_type.as_deref(); + if plan.taps <= 1 { + self.draw_cur_themed( + &cursor_sprites, + cursor_type, + plan.placement, + plan.size_px, + 1.0, + plan.clip, + ); + } else { + // Flou RÉEL, pas des copies discrètes : accumule les N échantillons dans un + // buffer ISOLÉ (transparent), pas directement sur la scène déjà composée. + self.ctx.ClearRenderTargetView(&self.accum_rtv, &[0.0, 0.0, 0.0, 0.0]); + self.ctx.OMSetRenderTargets(Some(&[Some(self.accum_rtv.clone())]), None); + for k in 0..plan.taps { + let f = k as f32 / (plan.taps - 1) as f32; + let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); + self.ctx.OMSetBlendState(&self.blend_add, Some(&[w, w, w, w]), 0xffffffff); self.draw_cur_themed( &cursor_sprites, cursor_type, - cur, - sz, + plan.prev_placement.lerp(plan.placement, f), + plan.size_px, 1.0, - cursor_clip_rect, + plan.clip, ); - } else { - let tp = t - trail_frames / FPS; - let prev = map(track.at(tp), [su0_p, sv0_p], [hu_p, hv_p], s_dst_prev) - .unwrap_or(cur); - // Flou RÉEL, pas des copies discrètes : accumule les N échantillons dans un - // buffer ISOLÉ (transparent), pas directement sur la scène déjà composée. - // BUG précédent : additionner directement sur `self.rtv` revient à AJOUTER - // la couleur du curseur (blanc) à ce qu'il y a déjà dessous — sur un fond - // clair, ajouter du blanc*petit-alpha ne change presque rien de visible - // (déjà proche du blanc) -> curseur quasi invisible. En accumulant d'abord - // dans un buffer à part (parti de zéro, même mécanisme que le motion blur - // écran de `compose_frame_mb`), la somme reste correctement normalisée - // (alpha final ~1 si les échantillons se recouvrent), puis on la composite - // sur la scène par un blend "over" classique — correct quel que soit le fond. - self.ctx.ClearRenderTargetView(&self.accum_rtv, &[0.0, 0.0, 0.0, 0.0]); - self.ctx.OMSetRenderTargets(Some(&[Some(self.accum_rtv.clone())]), None); - let w = 1.0 / taps as f32; - self.ctx.OMSetBlendState(&self.blend_add, Some(&[w, w, w, w]), 0xffffffff); - for k in 0..taps { - let f = k as f32 / (taps - 1) as f32; - self.draw_cur_themed( - &cursor_sprites, - cursor_type, - prev.lerp(cur, f), - sz, - 1.0, - cursor_clip_rect, - ); - } - // composite le buffer accumulé sur la scène (blend "over" normal, prémultiplié). - self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); - self.ctx.PSSetShaderResources(0, Some(&[Some(self.accum_srv.clone())])); - self.ctx.VSSetShader(&self.vs_fs, None); - self.ctx.PSSetShader(&self.ps_tex, None); - self.ctx.PSSetSamplers(0, Some(&[Some(self.sampler.clone())])); - let vp = D3D11_VIEWPORT { - TopLeftX: 0.0, TopLeftY: 0.0, - Width: self.rw(), Height: self.rh(), MinDepth: 0.0, MaxDepth: 1.0, - }; - self.ctx.RSSetViewports(Some(&[vp])); - self.ctx.OMSetBlendState(&self.blend, None, 0xffffffff); - self.ctx.Draw(3, 0); - self.ctx.PSSetShaderResources(0, Some(&[None])); - // restaure l'état de composition standard (VS/PS/topologie quad-strip) pour - // le dessin de la webcam qui suit juste après. - self.bind_compose_state(); } + // composite le buffer accumulé sur la scène (blend "over" normal, prémultiplié). + self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); + self.ctx.PSSetShaderResources(0, Some(&[Some(self.accum_srv.clone())])); + self.ctx.VSSetShader(&self.vs_fs, None); + self.ctx.PSSetShader(&self.ps_tex, None); + self.ctx.PSSetSamplers(0, Some(&[Some(self.sampler.clone())])); + let vp = D3D11_VIEWPORT { + TopLeftX: 0.0, TopLeftY: 0.0, + Width: self.rw(), Height: self.rh(), MinDepth: 0.0, MaxDepth: 1.0, + }; + self.ctx.RSSetViewports(Some(&[vp])); + self.ctx.OMSetBlendState(&self.blend, None, 0xffffffff); + self.ctx.Draw(3, 0); + self.ctx.PSSetShaderResources(0, Some(&[None])); + // restaure l'état de composition standard (VS/PS/topologie quad-strip) pour + // le dessin de la webcam qui suit juste après. + self.bind_compose_state(); } } } @@ -1538,7 +1970,13 @@ impl Compositor { scene_preset.as_deref(), Some("dual-frame") | Some("vertical-stack"), ); - if cfg.shadow && !webcam_is_block && shape_fade > 0.0 { + // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une + // ombre portée par un rectangle invisible se lit comme un artefact. + let is_cutout = matches!( + scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), + Some(e) if e.shader_code() == 1.0 + ) && self.webcam_mask.borrow().is_some(); + if cfg.shadow && !webcam_is_block && !is_cutout && shape_fade > 0.0 { let strength = WEBCAM_SHADOW_OPACITY * shape_fade; self.draw_shadow( w_dst, @@ -1549,6 +1987,38 @@ impl Compositor { strength, ); } + // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de + // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam + // invisible en détourage, donc tant que rien n'a été segmenté on dessine la piste + // telle quelle. C'est aussi ce qui rend le premier lancement gracieux. + let mask = self.webcam_mask.borrow(); + let effect = scene_ref + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .filter(|_| mask.is_some()) + .map(|e| (e.shader_code(), e)) + .filter(|(code, _)| *code > 0.0); + + // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra + // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement + // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de + // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et + // images y tombaient sur du noir, et le défaut EST une image. + let (effect_code, blur_intensity) = match effect { + Some((code, e)) if code > 2.5 => { + self.draw_webcam_bg(e.background.as_ref(), w_dst, w_px, w_radius); + (1.0, 0.0) + } + Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0)), + None => (0.0, 0.0), + }; + + if let Some(m) = mask.as_ref() { + // `draw_video` ne lie que les slots 0-1, donc le masque posé ici tient pour + // l'appel qui suit. Il est délié juste après pour ne pas fuir sur les calques + // d'annotation, qui utilisent eux aussi le slot 2 et au-delà. + self.ctx.PSSetShaderResources(3, Some(&[Some(m.srv.clone())])); + } self.draw_video( &LayerCB { dst: w_dst, @@ -1556,15 +2026,21 @@ impl Compositor { quad_px: w_px, radius_px: w_radius, mode: 0.0, + // `color.a` porte l'alpha du découpage (`color.a * personne`) ; le RGB n'est + // plus lu, le fond ayant déjà été peint sous la caméra. color: [0.0, 0.0, 0.0, 1.0], + fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], src_prev: [u0, sv0, u1, sv1], // src fixe (pas de zoom webcam) dst_prev: w_dst_prev, - mb: [mb_taps, 1.0, 1.0, 0.0], + mb: [mb_taps, mb_amount, 1.0, 0.0], ..Default::default() }, &wy, &wuv, ); + if mask.is_some() { + self.ctx.PSSetShaderResources(3, Some(&[None])); + } } // --- annotations : calque le plus haut, comme dans le DOM de la preview (le calque y est @@ -2398,6 +2874,86 @@ impl Compositor { mod tests { use super::*; + /// Preuve de bout en bout que `img_cache` est borné : charge TOUS les wallpapers livrés, + /// une frame par wallpaper — ce que fait le sélecteur quand on le parcourt — et vérifie que + /// le total reste sous le budget. + /// + /// Opt-in : il crée un vrai device D3D11, ce qu'aucun autre test de ce fichier ne fait (celui + /// juste en dessous s'en passe volontairement) et qu'un runner sans adaptateur ne peut pas + /// fournir. Même convention que le harnais visuel de la segmentation : + /// + /// set OPENSCREEN_CACHE_DEMO=1 && cargo test -p openscreen-compositor --release + /// img_cache_stays_under_budget -- --nocapture + /// + /// Les tests de `lru_evictions` couvrent la POLITIQUE ; celui-ci couvre le CÂBLAGE — que le + /// backend l'appelle vraiment, sur les bonnes tailles, et que le budget morde sur nos assets. + #[test] + fn img_cache_stays_under_budget() { + if std::env::var_os("OPENSCREEN_CACHE_DEMO").is_none() { + eprintln!("OPENSCREEN_CACHE_DEMO absent — saute (ce test demande un device D3D11)"); + return; + } + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .expect("racine du dépôt") + .join("public/wallpapers"); + let mut papers: Vec<_> = std::fs::read_dir(&root) + .expect("public/wallpapers") + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + matches!(p.extension().and_then(|e| e.to_str()), Some("jpg" | "jpeg" | "png")) + }) + .collect(); + papers.sort(); + assert!(papers.len() >= 10, "il faut plusieurs wallpapers pour que le budget morde"); + + let gpu = crate::d3d::Gpu::create_backend(crate::d3d::Backend::Hardware, false) + .expect("device D3D11"); + let comp = Compositor::new(&gpu).expect("compositeur"); + + let mut cumule = 0u64; + let mut pic = 0u64; + for path in &papers { + // Une frame par wallpaper : c'est le rythme du sélecteur, et c'est ce qui rend + // l'entrée précédente évinçable. Dans une même frame elle ne le serait pas. + comp.begin_image_frame(); + let p = path.to_string_lossy().to_string(); + let (_, w, h) = unsafe { comp.cached_image(&p) }.expect("chargement"); + cumule += w as u64 * h as u64 * 4; + let cache = comp.img_cache.borrow(); + let total: u64 = cache.values().map(|e| e.1 as u64 * e.2 as u64 * 4).sum(); + pic = pic.max(total); + eprintln!( + " {:<20} {:>5}x{:<5} | cache {:>2} entrées {:>4} Mo | cumulé sans éviction {:>5} Mo", + path.file_name().unwrap().to_string_lossy(), + w, + h, + cache.len(), + total / 1048576, + cumule / 1048576, + ); + } + eprintln!( + " + budget {} Mo | pic observé {} Mo | cumulé si rien n'était évincé {} Mo", + IMG_CACHE_BUDGET_BYTES / 1048576, + pic / 1048576, + cumule / 1048576, + ); + assert!( + pic <= IMG_CACHE_BUDGET_BYTES, + "le cache a dépassé son budget : {} Mo > {} Mo", + pic / 1048576, + IMG_CACHE_BUDGET_BYTES / 1048576 + ); + assert!( + cumule > IMG_CACHE_BUDGET_BYTES, + "sans éviction le total ({} Mo) doit dépasser le budget, sinon le test ne prouve rien", + cumule / 1048576 + ); + } + /// Le HLSL est compilé au démarrage du compositeur : jusqu'ici une faute dedans ne se voyait /// qu'à l'exécution, donc après un rebuild du natif ET un relancement de l'app. `D3DCompile` diff --git a/crates/compositor/src/cpu_frames_windows.rs b/crates/compositor/src/cpu_frames_windows.rs index c7a319308..790f23a7f 100644 --- a/crates/compositor/src/cpu_frames_windows.rs +++ b/crates/compositor/src/cpu_frames_windows.rs @@ -100,6 +100,11 @@ impl CpuFrames { } self.upload(w, h)?; + // `Decoder::seek_to` and `decode_forward_to` inspect the presentation frame returned by + // `present`, so it must carry the decoded frame's timing just like the macOS/Linux CPU + // paths. Leaving the allocation defaults here makes every non-H.264 frame look untimed. + (*self.present).pts = (*src).pts; + (*self.present).best_effort_timestamp = (*src).best_effort_timestamp; Ok(self.present) } diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index 621309e67..a34c2ae00 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -92,7 +92,7 @@ impl CursorTrack { /// Seul point de construction : garantit que `follow_samples` est toujours dérivé des /// échantillons courants. Une piste re-lissée (`smoothed`) recalcule donc aussi son suivi, /// pour que la caméra suive la trajectoire que l'utilisateur voit réellement. - fn new(samples: Vec<(f32, f32, f32)>, clicks: Vec, types: Vec<(f32, String)>) -> CursorTrack { + pub(crate) fn new(samples: Vec<(f32, f32, f32)>, clicks: Vec, types: Vec<(f32, String)>) -> CursorTrack { let follow_samples = smooth_follow_samples(&samples); CursorTrack { samples, follow_samples, clicks, types } } @@ -130,13 +130,18 @@ impl CursorTrack { if s["interactionType"].as_str() == Some("click") { clicks.push(t); } - // Seules les TRANSITIONS sont retenues — voir `types`. Les échantillons sans - // `cursorType` (macOS ne le tague pas toujours) n'interrompent pas l'état courant : - // c'est une absence d'information, pas un retour à la flèche. - if let Some(ct) = s["cursorType"].as_str() { - if types.last().map(|(_, prev)| prev.as_str()) != Some(ct) { - types.push((t, ct.to_string())); - } + // Seules les TRANSITIONS sont retenues — voir `types`. Le helper + // macOS rend nil hors texte/pointeur pour que le rendu retombe sur + // la flèche. Le sidecar stocke ça en JSON null ou omet la clé. + // Ignorer ces échantillons gardait le dernier type sémantique + // (`pointer`/`text`) : un thème restait collé après le retour à la + // flèche. Null / absence = reset vers `arrow`. + let ct = match s.get("cursorType") { + Some(v) => v.as_str().filter(|label| !label.is_empty()).unwrap_or("arrow"), + None => "arrow", + }; + if types.last().map(|(_, prev)| prev.as_str()) != Some(ct) { + types.push((t, ct.to_string())); } } samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); @@ -291,4 +296,49 @@ mod tests { assert_eq!(smoothed.type_at(0.1), Some("arrow")); assert_eq!(smoothed.type_at(0.7), Some("text")); } + + /// JSON null et une clé `cursorType` absente resetent vers la flèche, + /// au lieu de garder le dernier `pointer`/`text`. + #[test] + fn null_cursor_type_resets_to_arrow() { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let path = std::env::temp_dir().join(format!( + "openscreen-cursor-null-reset-{}-{}.json", + std::process::id(), + unique + )); + std::fs::write( + &path, + r#"{"samples":[ + {"timeMs":0,"cx":0.1,"cy":0.1,"cursorType":"pointer"}, + {"timeMs":100,"cx":0.2,"cy":0.2,"cursorType":null}, + {"timeMs":200,"cx":0.3,"cy":0.3,"cursorType":"pointer"}, + {"timeMs":300,"cx":0.4,"cy":0.4} + ]}"#, + ) + .expect("write temp sidecar"); + let path_str = path.to_str().expect("utf-8 temp path"); + let track = CursorTrack::load(path_str, 0.0, 1.0).expect("load sidecar"); + let _ = std::fs::remove_file(&path); + + assert_eq!(track.type_at(0.00), Some("pointer")); + assert_eq!( + track.type_at(0.10), + Some("arrow"), + "JSON null must reset to arrow" + ); + assert_eq!( + track.type_at(0.20), + Some("pointer"), + "pointer after null must hold until the omitted-key sample" + ); + assert_eq!( + track.type_at(0.30), + Some("arrow"), + "omitted cursorType after pointer must reset to arrow independently" + ); + } } diff --git a/crates/compositor/src/d3d_linux.rs b/crates/compositor/src/d3d_linux.rs index 7c344d831..3d74edf65 100644 --- a/crates/compositor/src/d3d_linux.rs +++ b/crates/compositor/src/d3d_linux.rs @@ -139,24 +139,36 @@ async fn create_async(want: Backend) -> Result { info.name ); } - let (device, queue) = adapter - .request_device( - &wgpu::DeviceDescriptor { - label: Some("openscreen-linux"), - required_features: wgpu::Features::empty(), - required_limits: wgpu::Limits::default(), - memory_hints: wgpu::MemoryHints::default(), - }, - None, - ) - .await - .context("request_device a echoue")?; + let desc = wgpu::DeviceDescriptor { + label: Some("openscreen-linux"), + required_features: wgpu::Features::empty(), + required_limits: wgpu::Limits::default(), + memory_hints: wgpu::MemoryHints::default(), + }; + // Ouvre le device AVEC les extensions de memoire externe si la machine les a, + // et retombe sur le chemin standard sinon. Le repli couvre un hote sans + // pilote Vulkan utilisable, ou un pilote sans memoire externe ; sur cette + // machine il n'est plus atteint depuis qu'on n'exige que deux extensions. + let (device, queue, dmabuf_export) = match open_device_with_dmabuf_export(&adapter, &desc) { + Some((d, q)) => (d, q, true), + None => { + let (d, q) = adapter + .request_device(&desc, None) + .await + .context("request_device a echoue")?; + (d, q, false) + } + }; // Windows loggue son repli (`d3d_windows.rs`), Linux ne loggait rien : un hote // tombe sur lavapipe rendait a quelques fps sans que rien -- ni log, ni rapport // de bug -- ne permette de l'etablir a distance. eprintln!( - "[d3d] adaptateur Vulkan : {} ({:?}, {:?}) -> backend {:?}", - info.name, info.device_type, info.backend, got + "[d3d] adaptateur Vulkan : {} ({:?}, {:?}) -> backend {:?}, export dmabuf {}", + info.name, + info.device_type, + info.backend, + got, + if dmabuf_export { "actif" } else { "indisponible" } ); Ok(Gpu { device, @@ -394,3 +406,114 @@ mod tests { } } } + +/// Ouvre le `VkDevice` en AJOUTANT les extensions qui permettent d'exporter une +/// image en dmabuf, et rend `None` si la machine ne les a pas toutes. +/// +/// POURQUOI PASSER SOUS wgpu. `request_device` n'a aucun moyen de demander une +/// extension Vulkan : wgpu n'active que ce que ses propres `Features` imposent. +/// Or l'export dmabuf n'a pas de `Feature` equivalente. Le seul point d'entree +/// est donc de construire le device soi-meme et de le rendre a wgpu via +/// `create_device_from_hal`. +/// +/// CE QUE CETTE FONCTION NE FAIT PAS. Elle n'exporte rien : elle rend seulement +/// l'export POSSIBLE plus tard. Tant que personne n'appelle `vkGetMemoryFdKHR`, +/// activer ces extensions ne change ni le rendu ni les performances -- c'est +/// justement ce qui permet de la livrer seule et de la verifier seule. +/// +/// LE REPLI EST LE CAS NORMAL, PAS L'EXCEPTION. Le rasteriseur logiciel +/// (lavapipe) n'expose pas `VK_EXT_image_drm_format_modifier`, et une machine +/// sans pilote GPU utilisable non plus. Rendre `None` doit donc rester +/// silencieux et sans consequence : l'appelant repart sur `request_device`. +#[cfg(target_os = "linux")] +fn open_device_with_dmabuf_export( + adapter: &wgpu::Adapter, + desc: &wgpu::DeviceDescriptor<'_>, +) -> Option<(wgpu::Device, wgpu::Queue)> { + use std::ffi::CStr; + + // Les deux qu'il faut EN PLUS de ce que wgpu demande deja. `dma_buf` depend + // de `external_memory_fd` ; ensemble elles suffisent a exporter la memoire + // d'un buffer sous forme de descripteur dmabuf. + // + // PAS `VK_EXT_image_drm_format_modifier`. Il ne servirait qu'a exporter une + // IMAGE, dont la disposition en memoire depend du pavage et doit donc etre + // decrite au consommateur. Ce qu'on exporte ici est le buffer de staging que + // le compositeur remplit deja par `copy_texture_to_buffer` : lineaire par + // construction, avec des pitches qu'on choisit. Il n'y a aucun pavage a + // decrire, donc rien a demander au pilote. + // + // L'exiger etait une erreur mesurable, pas une precaution : c'est + // precisement l'extension que le rasteriseur logiciel n'expose pas, donc + // reclamer les trois refusait l'export a des machines parfaitement capables + // de le faire. + const WANTED: [&CStr; 2] = [c"VK_KHR_external_memory_fd", c"VK_EXT_external_memory_dma_buf"]; + + unsafe { + adapter.as_hal::(|hal_adapter| { + let hal_adapter = hal_adapter?; + let phys = hal_adapter.raw_physical_device(); + let instance = hal_adapter.shared_instance().raw_instance(); + + // Refuser tot plutot que d'echouer a `vkCreateDevice` : une extension + // absente y devient une erreur opaque. + let available = instance.enumerate_device_extension_properties(phys).ok()?; + let has = |name: &CStr| { + available + .iter() + .any(|e| e.extension_name_as_c_str() == Ok(name)) + }; + if !WANTED.iter().all(|n| has(n)) { + return None; + } + + // Aux extensions de wgpu, pas a la place : en omettre une casserait + // le rendu, pas l'export. + let mut exts = hal_adapter.required_device_extensions(desc.required_features); + exts.extend_from_slice(&WANTED); + let ext_ptrs: Vec<*const std::os::raw::c_char> = + exts.iter().map(|e| e.as_ptr()).collect(); + + // La famille 0 n'est PAS garantie graphique. Elle l'est sur la + // plupart des pilotes, ce qui rend l'erreur invisible jusqu'a la + // machine ou elle ne l'est pas — et la panne serait alors un device + // qui s'ouvre puis ne sait rien dessiner. + let families = instance.get_physical_device_queue_family_properties(phys); + let family_index = families + .iter() + .position(|f| f.queue_flags.contains(ash::vk::QueueFlags::GRAPHICS))? + as u32; + let queue_prio = [1.0f32]; + let queue_info = ash::vk::DeviceQueueCreateInfo::default() + .queue_family_index(family_index) + .queue_priorities(&queue_prio); + let queue_infos = [queue_info]; + + // `physical_device_features` porte les activations que wgpu attend + // (elles vivent dans des structures chainees) : les reprendre telles + // quelles est ce qui garantit que le device rendu est celui que wgpu + // aurait construit, extensions en plus. + let mut phys_features = + hal_adapter.physical_device_features(&exts, desc.required_features); + let info = phys_features.add_to_device_create( + ash::vk::DeviceCreateInfo::default() + .queue_create_infos(&queue_infos) + .enabled_extension_names(&ext_ptrs), + ); + let raw_device = instance.create_device(phys, &info, None).ok()?; + + let open = hal_adapter + .device_from_raw( + raw_device, + None, + &exts, + desc.required_features, + &desc.memory_hints, + family_index, + 0, + ) + .ok()?; + adapter.create_device_from_hal(open, desc, None).ok() + }) + } +} diff --git a/crates/compositor/src/export_probe.rs b/crates/compositor/src/export_probe.rs new file mode 100644 index 000000000..667138042 --- /dev/null +++ b/crates/compositor/src/export_probe.rs @@ -0,0 +1,144 @@ +//! Sondes de temps par étage pour l'export, activées par `OPENSCREEN_EXPORT_PROFILE=1`. +//! +//! Le but est de répondre à UNE question — où part le temps d'un export — sans avoir à +//! croire une intuition. Chaque étage accumule des nanosecondes et un compte d'appels ; +//! `report` imprime le tableau sur stderr à la fin de la marche. +//! +//! # Coût quand c'est éteint +//! +//! `scope()` lit un `OnceLock` et, si la sonde est éteinte, ne prend AUCUNE horloge : +//! le `Scope` rendu porte `None` et son `Drop` ne fait rien. Allumée, elle coûte deux +//! `Instant::now()` (un `mach_absolute_time` chacun, ~20 ns sur Apple Silicon) et un +//! `fetch_add` relaxé par étage et par frame. +//! +//! # Ce que les nombres veulent dire, et ne veulent pas dire +//! +//! Les étages sont mesurés là où le CPU les appelle, pas là où le GPU les exécute. Metal +//! est asynchrone : `compose_frame` ne fait que soumettre, et l'attente de TOUT le travail +//! GPU de la frame tombe dans `gpu_wait`. Lire `compose` comme « le coût de la composition » +//! est donc faux — c'est le coût de la CONSTRUIRE, pas de la rendre. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::OnceLock; +use std::time::Instant; + +#[derive(Clone, Copy)] +pub enum Stage { + DecodeScreen = 0, + DecodeWebcam = 1, + Compose = 2, + VtGetBuffer = 3, + Nv12Passes = 4, + GpuWait = 5, + SendFrame = 6, + DrainMux = 7, + Progress = 8, + Finalize = 9, +} + +const N: usize = 10; + +const NAMES: [&str; N] = [ + "decode.screen", + "decode.webcam", + "compose.submit", + "vt.get_buffer", + "nv12.passes", + "gpu.wait", + "enc.send_frame", + "mux.drain", + "progress.cb", + "finalize", +]; + +static NANOS: [AtomicU64; N] = [ + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), +]; +static COUNT: [AtomicU64; N] = [ + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), +]; + +static ENABLED: OnceLock = OnceLock::new(); + +pub fn enabled() -> bool { + *ENABLED.get_or_init(|| { + matches!( + std::env::var("OPENSCREEN_EXPORT_PROFILE").ok().as_deref(), + Some("1") | Some("true") + ) + }) +} + +pub struct Scope { + stage: usize, + t0: Option, +} + +impl Drop for Scope { + fn drop(&mut self) { + if let Some(t0) = self.t0 { + NANOS[self.stage].fetch_add(t0.elapsed().as_nanos() as u64, Ordering::Relaxed); + COUNT[self.stage].fetch_add(1, Ordering::Relaxed); + } + } +} + +/// Ouvre une sonde sur `stage`. Le temps est compté jusqu'au `Drop` du `Scope` rendu. +pub fn scope(stage: Stage) -> Scope { + Scope { + stage: stage as usize, + t0: if enabled() { Some(Instant::now()) } else { None }, + } +} + +/// Imprime le tableau sur stderr. `wall_s` est le mur total de la fonction d'export, ce qui +/// permet de voir ce que les étages NE couvrent pas. +pub fn report(wall_s: f64, frames: u64) { + if !enabled() { + return; + } + let total_ns: u64 = (0..N).map(|i| NANOS[i].load(Ordering::Relaxed)).sum(); + eprintln!("[profile] {frames} frames en {wall_s:.3} s ({:.1} fps)", frames as f64 / wall_s.max(1e-9)); + eprintln!("[profile] {:<16} {:>10} {:>9} {:>8} {:>7}", "étage", "total (s)", "µs/frame", "% mur", "appels"); + let mut rows: Vec = (0..N).collect(); + rows.sort_by_key(|&i| std::cmp::Reverse(NANOS[i].load(Ordering::Relaxed))); + for i in rows { + let ns = NANOS[i].load(Ordering::Relaxed); + let c = COUNT[i].load(Ordering::Relaxed); + if c == 0 { + continue; + } + eprintln!( + "[profile] {:<16} {:>10.3} {:>9.1} {:>7.1}% {:>7}", + NAMES[i], + ns as f64 / 1e9, + ns as f64 / 1e3 / c as f64, + 100.0 * (ns as f64 / 1e9) / wall_s.max(1e-9), + c + ); + } + eprintln!( + "[profile] {:<16} {:>10.3} {:>9} {:>7.1}%", + "SOMME sondes", + total_ns as f64 / 1e9, + "", + 100.0 * (total_ns as f64 / 1e9) / wall_s.max(1e-9) + ); + eprintln!( + "[profile] {:<16} {:>10.3} {:>9} {:>7.1}% <- ce que les sondes ne couvrent pas", + "non sondé", + wall_s - total_ns as f64 / 1e9, + "", + 100.0 * (wall_s - total_ns as f64 / 1e9) / wall_s.max(1e-9) + ); +} + +/// Remet tous les compteurs à zéro. Un même process peut enchaîner deux exports. +pub fn reset() { + for i in 0..N { + NANOS[i].store(0, Ordering::Relaxed); + COUNT[i].store(0, Ordering::Relaxed); + } +} diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index 655b746b7..03a838437 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -379,7 +379,7 @@ pub(crate) fn cursor_sprite_dst(center: [f32; 2], w: f32, h: f32, hotspot: [f32; /// donc pas seulement sa position qu'il faut projeter mais son sprite entier : autrement il se /// lit comme un autocollant plat posé sur une scène en perspective. #[derive(Clone, Copy)] -pub(crate) enum CursorPlacement { +pub enum CursorPlacement { /// Écran droit : centre en coordonnées sortie 0..1. Upright { center: [f32; 2] }, /// Écran incliné : position 0..1 DANS le plan, plus de quoi projeter les coins du sprite. @@ -732,6 +732,7 @@ pub struct FrameGeometryInput<'a> { pub struct FrameGeometry { pub scene_preset: Option, pub mb_taps: f32, + pub mb_amount: f32, pub source_t: f32, pub zoom_rotation: [f32; 3], pub padding_scale: f32, @@ -862,6 +863,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let mb_taps = scene .map(|s| 1.0 + s.effects.motion_blur.clamp(0.0, 1.0) * 15.0) .unwrap_or(cfg.mblur_n as f32); + let mb_amount = scene + .map(|s| s.effects.motion_blur.clamp(0.0, 1.0)) + .unwrap_or(if cfg.mblur_n > 1 { 1.0 } else { 0.0 }); // Zoom regions + Full Camera : filtrées en amont pour le clip actif et échantillonnées // dans le même référentiel source que le PTS du décodeur écran. @@ -1165,6 +1169,7 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { FrameGeometry { scene_preset, mb_taps, + mb_amount, source_t, zoom_rotation, padding_scale, @@ -1280,16 +1285,32 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option Option f32 { + if taps <= 1 { + return 1.0; + } + let t = k as f32 / (taps - 1) as f32; + let ramp = 0.25 + 0.75 * t; + let sum = taps as f32 * 0.625; + ramp / sum +} + +/// Les clés à évincer d'un cache de textures pour repasser sous `budget`, la moins récemment +/// utilisée d'abord. `entries` porte `(clé, octets, tick d'usage)`. +/// +/// `protect_from` est le tick au DÉBUT DE LA FRAME EN COURS : toute entrée touchée depuis est +/// intouchable. Protéger la seule entrée qu'on vient de poser ne suffit pas — une frame échantillonne +/// plusieurs textures (fond d'écran, fond de caméra, sprites de curseur), et évincer l'une d'elles +/// parce qu'une autre vient d'arriver la ferait recharger à la frame suivante, puis rechasser la +/// suivante : le cache se mettrait à battre au lieu de servir. Un décodage mesuré à 129 ms en +/// release contre les ~3,5 ms d'une frame, c'est un échange qu'aucun budget mémoire ne justifie. +/// +/// Si le jeu actif dépasse à lui seul le budget, la fonction s'arrête AU-DESSUS du budget plutôt +/// que d'y toucher. Dépasser est le moindre mal. +/// +/// Partagé plutôt que recopié dans chaque backend, pour la raison qui vaut pour tout ce module : +/// trois copies d'une politique d'éviction finiraient par diverger sans que rien ne le dise. +pub fn lru_evictions(entries: &[(String, u64, u64)], budget: u64, protect_from: u64) -> Vec { + let mut total: u64 = entries.iter().map(|(_, bytes, _)| *bytes).sum(); + if total <= budget { + return Vec::new(); + } + let mut candidates: Vec<&(String, u64, u64)> = + entries.iter().filter(|(_, _, tick)| *tick < protect_from).collect(); + candidates.sort_by_key(|(_, _, tick)| *tick); + let mut out = Vec::new(); + for (key, bytes, _) in candidates { + if total <= budget { + break; + } + total -= bytes; + out.push(key.clone()); + } + out +} + #[cfg(test)] mod tests { + use super::lru_evictions; + + /// `(clé, octets, tick)` — le tick croît avec l'usage, donc le plus petit est le plus ancien. + fn e(key: &str, mb: u64, tick: u64) -> (String, u64, u64) { + (key.to_string(), mb * 1024 * 1024, tick) + } + + const BUDGET: u64 = 512 * 1024 * 1024; + + #[test] + fn evicts_nothing_while_under_budget() { + assert!(lru_evictions(&[e("a", 100, 1), e("b", 100, 2)], BUDGET, 2).is_empty()); + } + + /// La plus ancienne part d'abord, et on s'arrête DÈS qu'on repasse sous le budget : évincer + /// au-delà ne rendrait que des rechargements. + #[test] + fn evicts_oldest_first_and_stops_at_the_budget() { + let entries = [e("vieux", 100, 1), e("moyen", 100, 2), e("neuf", 100, 9)]; + assert_eq!(lru_evictions(&entries, 250 * 1024 * 1024, 9), vec!["vieux".to_string()]); + } + + /// TOUT le jeu actif de la frame est protégé, pas seulement la dernière entrée posée. Une + /// frame qui échantillonne un fond d'écran ET un fond de caméra ne doit pas voir le premier + /// évincé parce que le second vient d'arriver — sinon les deux se chassent l'un l'autre à + /// chaque frame. + #[test] + fn protects_every_texture_used_this_frame() { + // frame commencée au tick 5 : `ecran` et `camera` servent tous deux maintenant. + let entries = [e("vieux", 100, 2), e("ecran", 400, 5), e("camera", 400, 6)]; + assert_eq!(lru_evictions(&entries, BUDGET, 5), vec!["vieux".to_string()]); + } + + /// Jeu actif plus gros que le budget : on rend ce qu'on peut et on reste au-dessus, plutôt que + /// de faire disparaître des textures dont cette frame a besoin. + #[test] + fn gives_up_rather_than_evicting_the_active_set() { + let entries = [e("a", 100, 1), e("actif", 900, 5)]; + assert_eq!(lru_evictions(&entries, 256 * 1024 * 1024, 5), vec!["a".to_string()]); + } + use super::*; /// La scène de référence du golden : un cas qui exerce le padding, le crop, le zoom, @@ -1908,4 +2019,110 @@ mod tests { let uv = webcam_source_rect([100.0, 100.0], [100.0, 100.0], Some(crop), 0.50 / 0.60); assert_rect(uv, [0.25, 0.20, 0.75, 0.80]); } + + #[test] + fn cursor_tap_weight_sums_to_one_and_is_monotonically_increasing() { + assert_eq!(cursor_tap_weight(0, 1), 1.0); + + for taps in [2, 4, 8, 11, 16] { + let mut sum = 0.0; + let mut prev_w = 0.0; + for k in 0..taps { + let w = cursor_tap_weight(k, taps); + assert!(w > 0.0, "poids positif"); + if k > 0 { + assert!(w > prev_w, "tête plus marquée que la queue : {w} > {prev_w}"); + } + prev_w = w; + sum += w; + } + assert!((sum - 1.0).abs() < 1e-5, "somme des poids = 1.0 pour taps={taps}, got {sum}"); + } + } + + #[test] + fn plan_cursor_motion_blur_adaptive_and_stationary() { + let cfg = crate::config::all().pop().expect("cfg"); + let track_immobile = crate::cursor::CursorTrack::new( + vec![(0.0, 0.5, 0.5), (2.0, 0.5, 0.5)], + vec![], + vec![], + ); + let track_moving = crate::cursor::CursorTrack::new( + vec![(0.0, 0.1, 0.1), (1.0, 0.9, 0.9)], + vec![], + vec![], + ); + let scene = zoomed_golden_scene(); + let fg = FrameGeometry { + scene_preset: None, + mb_taps: 1.0, + mb_amount: 0.0, + source_t: 0.0, + zoom_rotation: [0.0, 0.0, 0.0], + padding_scale: 1.0, + cut: [0.0, 0.0, 1.0, 1.0], + s_dst: [0.0, 0.0, 1.0, 1.0], + s_dst_prev: [0.0, 0.0, 1.0, 1.0], + s_ann: [0.0, 0.0, 1.0, 1.0], + s_radius: 0.0, + frame_min_px: 1080.0, + w_dst: [0.0, 0.0, 0.0, 0.0], + w_dst_prev: [0.0, 0.0, 0.0, 0.0], + w_px: [0.0, 0.0], + w_radius: 0.0, + shape_fade: 0.0, + }; + + // 1. Curseur immobile avec blur actif -> taps = 1 + let live_with_blur = LiveParams { + cursor_motion_blur: 0.8, + ..LiveParams::default() + }; + let input_immobile = CursorPlanInput { + render_px: [1920.0, 1080.0], + u_max: 1.0, + v_max: 1.0, + cfg: &cfg, + live: live_with_blur, + scene: Some(&scene), + track: &track_immobile, + t: 0.5, + }; + let plan = plan_cursor(&fg, &input_immobile).expect("plan cursor"); + assert_eq!(plan.taps, 1, "curseur immobile doit rester à 1 tap"); + + // 2. Curseur avec blur = 0 -> taps = 1 + let live_no_blur = LiveParams { + cursor_motion_blur: 0.0, + ..LiveParams::default() + }; + let input_no_blur = CursorPlanInput { + render_px: [1920.0, 1080.0], + u_max: 1.0, + v_max: 1.0, + cfg: &cfg, + live: live_no_blur, + scene: Some(&scene), + track: &track_moving, + t: 0.5, + }; + let plan = plan_cursor(&fg, &input_no_blur).expect("plan cursor"); + assert_eq!(plan.taps, 1, "blur=0 doit donner taps = 1"); + + // 3. Curseur en mouvement rapide avec blur -> taps adaptatifs entre 2 et 16 + let input_moving = CursorPlanInput { + render_px: [1920.0, 1080.0], + u_max: 1.0, + v_max: 1.0, + cfg: &cfg, + live: live_with_blur, + scene: Some(&scene), + track: &track_moving, + t: 0.5, + }; + let plan = plan_cursor(&fg, &input_moving).expect("plan cursor"); + assert!(plan.taps >= 2 && plan.taps <= 16, "taps adaptatifs dans [2, 16], got {}", plan.taps); + } } + diff --git a/crates/compositor/src/gif_export.rs b/crates/compositor/src/gif_export.rs index a30a43007..c74adb090 100644 --- a/crates/compositor/src/gif_export.rs +++ b/crates/compositor/src/gif_export.rs @@ -254,7 +254,7 @@ fn export_gif_inner( let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); screen_decs.insert(clips[0].screen.clone(), unsafe { - Decoder::open(&clips[0].screen, gpu)? + Decoder::open_for_export(&clips[0].screen, gpu)? }); let frames = unsafe { diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index d8972e7d1..ac9c93bab 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -28,8 +28,10 @@ //! — c'est précisément ce qui rend le port Metal possible (cf. PR #162). pub mod audio; +pub mod audio_jobs; pub mod config; pub mod cursor; +pub mod export_probe; pub mod ffi; pub mod frame_geometry; pub mod gif_export; @@ -40,6 +42,10 @@ pub mod regions; // n'est spécifique à Linux. pub mod remux; pub mod scene; +// Segmentation du sujet webcam (masque -> `t3`). Le module compile toujours ; sans la feature +// `segmentation` ses deux entrées échouent proprement, ce qui garde le reste du crate +// indépendant du choix de packaging d'ONNX Runtime. +pub mod segmentation; pub mod text_anim; pub mod text_plate; pub(crate) mod timeline_walk; diff --git a/crates/compositor/src/linux_frames.rs b/crates/compositor/src/linux_frames.rs index dcb99bedb..044c05b7e 100644 --- a/crates/compositor/src/linux_frames.rs +++ b/crates/compositor/src/linux_frames.rs @@ -35,13 +35,22 @@ use crate::ffi::{ /// genere pas les `SWS_*`, ce sont des macros). const SWS_POINT: i32 = 0x10; -/// Une frame decodee presentee au compositor sous forme de deux textures wgpu : -/// plane Y (`R8Unorm`, `w x h`) et plane UV entrelacee (`Rg8Unorm`, -/// `(w/2) x (h/2)`). Equivalent NV12-split de la `ID3D11Texture2D` NV12 (D3D11) -/// / du CVPixelBuffer (macOS). +/// Une frame decodee presentee au compositor sous forme de TROIS textures wgpu +/// `R8Unorm` : Y en `w x h`, U et V en `(w/2) x (h/2)`. Pendant Linux de la +/// `ID3D11Texture2D` NV12 (D3D11) / du CVPixelBuffer (macOS), qui eux portent un +/// plan de chroma entrelace parce que leur decodeur materiel le rend ainsi. +/// +/// POURQUOI TROIS PLANS ET PAS UN UV ENTRELACE. Le decodeur software rend du +/// YUV420P, ou U et V sont DEJA deux plans distincts. Les entrelacer en NV12 +/// demandait un `sws_scale` CPU par frame et par flux — deux fois par frame de +/// sortie sur une scene avec webcam — pour produire une disposition que le GPU +/// echantillonne tout aussi bien en deux textures. Les uploader tels quels +/// supprime cette conversion sans changer un pixel : c'etait un entrelacement, +/// pas un reechantillonnage. pub(crate) struct VkFrameTex { pub y: wgpu::Texture, - pub uv: wgpu::Texture, + pub u: wgpu::Texture, + pub v: wgpu::Texture, pub width: u32, pub height: u32, } @@ -107,24 +116,34 @@ impl CpuFrames { if w <= 0 || h <= 0 { bail!("frame decodee sans dimensions ({w}x{h})"); } - self.ensure_sws(w, h, (*src).format)?; - self.ensure_nv12(w, h)?; self.ensure_textures(w as u32, h as u32)?; - let converted = sws_scale( - self.sws, - (*src).data.as_ptr() as *const *const u8, - (*src).linesize.as_ptr(), - 0, - h, - (*self.nv12).data.as_ptr(), - (*self.nv12).linesize.as_ptr(), - ); - if converted <= 0 { - bail!("sws_scale a converti {converted} lignes"); + // CHEMIN RAPIDE : le decodeur rend deja du YUV420P (c'est le cas de tout + // h264 4:2:0, donc de tout ce que cette app enregistre), et c'est + // exactement la disposition que les trois textures attendent. Rien a + // convertir : on uploade les plans du decodeur tels quels. + if (*src).format == AVPixelFormat::AV_PIX_FMT_YUV420P as i32 { + self.upload_planes(src)?; + } else { + // REPLI : format exotique (4:2:2, 10 bits, un import quelconque). + // `sws_scale` ramene en YUV420P — pas en NV12 : la cible n'a plus de + // plan entrelace — et on uploade le resultat par le meme chemin. + self.ensure_sws(w, h, (*src).format)?; + self.ensure_nv12(w, h)?; + let converted = sws_scale( + self.sws, + (*src).data.as_ptr() as *const *const u8, + (*src).linesize.as_ptr(), + 0, + h, + (*self.nv12).data.as_ptr(), + (*self.nv12).linesize.as_ptr(), + ); + if converted <= 0 { + bail!("sws_scale a converti {converted} lignes"); + } + self.upload_planes(self.nv12)?; } - - self.upload()?; self.attach_carrier(w, h)?; // Contrat lu par le compositor : sentinel + timestamps recopies (sinon la // timeline se croit a t=0). @@ -148,14 +167,14 @@ impl CpuFrames { src_fmt as AVPixelFormat::Type, w, h, - AVPixelFormat::AV_PIX_FMT_NV12, + AVPixelFormat::AV_PIX_FMT_YUV420P, SWS_POINT, ptr::null_mut(), ptr::null_mut(), ptr::null(), ); if self.sws.is_null() { - bail!("sws_getContext {w}x{h} fmt {src_fmt} -> NV12"); + bail!("sws_getContext {w}x{h} fmt {src_fmt} -> YUV420P"); } self.sws_key = key; Ok(()) @@ -164,14 +183,14 @@ impl CpuFrames { unsafe fn ensure_nv12(&mut self, w: i32, h: i32) -> Result<()> { if (*self.nv12).width == w && (*self.nv12).height == h - && (*self.nv12).format == AVPixelFormat::AV_PIX_FMT_NV12 as i32 + && (*self.nv12).format == AVPixelFormat::AV_PIX_FMT_YUV420P as i32 { return Ok(()); } av_frame_unref(self.nv12); (*self.nv12).width = w; (*self.nv12).height = h; - (*self.nv12).format = AVPixelFormat::AV_PIX_FMT_NV12 as i32; + (*self.nv12).format = AVPixelFormat::AV_PIX_FMT_YUV420P as i32; if av_frame_get_buffer(self.nv12, 32) < 0 { bail!("av_frame_get_buffer NV12 {w}x{h}"); } @@ -202,23 +221,28 @@ impl CpuFrames { usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, view_formats: &[], }); - let uv = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("nv12-uv"), - size: wgpu::Extent3d { - width: dims.0 / 2, - height: dims.1 / 2, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rg8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); + let mut chroma = |label| { + self.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: dims.0 / 2, + height: dims.1 / 2, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }) + }; + let u = chroma("yuv420p-u"); + let v = chroma("yuv420p-v"); self.tex = Some(Box::new(VkFrameTex { y, - uv, + u, + v, width: dims.0, height: dims.1, })); @@ -229,53 +253,52 @@ impl CpuFrames { /// Upload du NV12 swscale dans les deux textures wgpu. `linesize[0]/[1]` sont /// les strides memoire (paddes SIMD par swscale), passes tels quels a /// `bytes_per_row`. - unsafe fn upload(&mut self) -> Result<()> { + /// Uploade les trois plans YUV420P de `f` dans les trois textures. `f` est + /// soit la frame du decodeur (chemin rapide), soit la sortie de swscale + /// (repli) : la disposition est la meme, seule la provenance change. + unsafe fn upload_planes(&mut self, f: *mut AVFrame) -> Result<()> { let tex = match self.tex.as_ref() { Some(t) => t, None => bail!("upload avant ensure_textures"), }; - let y_stride = (*self.nv12).linesize[0] as usize; - let uv_stride = (*self.nv12).linesize[1] as usize; - let y_size = y_stride * tex.height as usize; - let uv_size = uv_stride * tex.height.div_ceil(2) as usize; - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture: &tex.y, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - std::slice::from_raw_parts((*self.nv12).data[0], y_size), - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(y_stride as u32), - rows_per_image: Some(tex.height), - }, - wgpu::Extent3d { - width: tex.width, - height: tex.height, - depth_or_array_layers: 1, - }, - ); - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture: &tex.uv, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - std::slice::from_raw_parts((*self.nv12).data[1], uv_size), - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(uv_stride as u32), - rows_per_image: Some(tex.height / 2), - }, - wgpu::Extent3d { - width: tex.width / 2, - height: tex.height / 2, - depth_or_array_layers: 1, - }, - ); + // LES DIMENSIONS DE TEXTURE SONT ARRONDIES AU PAIR, PAS LES PLANS. + // `ensure_textures` arrondit pour que le chroma 4:2:0 tombe juste, mais + // le decodeur, lui, alloue au visible : lire `stride * hauteur_arrondie` + // depasse le plan d'une ligne sur une source de hauteur impaire. On lit + // donc le VISIBLE et on laisse la derniere ligne de la texture telle + // qu'elle est — elle n'existe que pour l'alignement. + let (vw, vh) = ((*f).width.max(0) as u32, (*f).height.max(0) as u32); + let (vw, vh) = (vw.min(tex.width), vh.min(tex.height)); + let (cw, chh) = (vw.div_ceil(2), vh.div_ceil(2)); + for (plane, texture, pw, ph) in [ + (0usize, &tex.y, vw, vh), + (1, &tex.u, cw, chh), + (2, &tex.v, cw, chh), + ] { + let stride = (*f).linesize[plane] as usize; + if stride == 0 || (*f).data[plane].is_null() { + bail!("plan YUV {plane} absent (linesize={stride})"); + } + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + std::slice::from_raw_parts((*f).data[plane], stride * ph as usize), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(stride as u32), + rows_per_image: Some(ph), + }, + wgpu::Extent3d { + width: pw, + height: ph, + depth_or_array_layers: 1, + }, + ); + } Ok(()) } @@ -292,7 +315,8 @@ impl CpuFrames { } (*self.present).data[0] = pack_carrier(Box::new(VkFrameTex { y: tex.y.clone(), - uv: tex.uv.clone(), + u: tex.u.clone(), + v: tex.v.clone(), width: tex.width, height: tex.height, })); @@ -322,14 +346,16 @@ pub(crate) unsafe fn carrier_dims(frame: *const AVFrame) -> (u32, u32) { /// depuis le carrier `frame.data[0]`. Appele par `compositor_linux`. pub(crate) unsafe fn nv12_planes( frame: *const AVFrame, -) -> Result<(wgpu::TextureView, wgpu::TextureView)> { +) -> Result<(wgpu::TextureView, wgpu::TextureView, wgpu::TextureView)> { if (*frame).data[0].is_null() { bail!("nv12_planes: carrier nul dans data[0]"); } let tex = unpack_carrier((*frame).data[0]); + let d = wgpu::TextureViewDescriptor::default(); Ok(( - tex.y.create_view(&wgpu::TextureViewDescriptor::default()), - tex.uv.create_view(&wgpu::TextureViewDescriptor::default()), + tex.y.create_view(&d), + tex.u.create_view(&d), + tex.v.create_view(&d), )) } diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 552d2264a..9d91fe643 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -1096,6 +1096,29 @@ type PendingPrefetch = (usize, std::sync::mpsc::Receiver> /// décodeurs ouvertes plus longtemps que nécessaire. const PREFETCH_LEAD_SEC: f64 = 0.75; +/// Durée pendant laquelle la boucle continue de recomposer après un changement en pause, le +/// temps qu'un effet asynchrone (segmentation webcam) livre son résultat. Généreuse : à +/// l'échelle d'une pause, une demi-seconde de recomposes ne coûte rien, alors qu'une fenêtre +/// trop courte laisse l'effet invisible sur une machine lente — exactement le bug d'origine. +const SETTLE_WINDOW: Duration = Duration::from_millis(500); +/// Cadence des recomposes dans cette fenêtre : celle de la segmentation (`SEGMENTATION_HZ`), +/// pas celle de la boucle — recomposer à 250 Hz n'accélérerait pas une inférence limitée à 30 Hz. +const SETTLE_STEP: Duration = Duration::from_millis(33); + +/// Ouvre (ou rouvre) la fenêtre de stabilisation. Les deux appelants — changement en pause et +/// seek en pause — doivent poser la MÊME paire : un `last_settle` oublié ferait recomposer à la +/// cadence de la boucle au lieu de celle de la segmentation. +fn open_settle_window(now: Instant) -> (Option, Instant) { + (Some(now + SETTLE_WINDOW), now) +} + +/// Faut-il recomposer alors que RIEN n'a changé ? Oui tant que la fenêtre de stabilisation +/// court et que la cadence le permet. Extrait de la boucle pour être vérifiable sans GPU. +fn should_settle(now: Instant, settle_until: Option, last_settle: Instant) -> bool { + settle_until.is_some_and(|deadline| now < deadline) + && now.duration_since(last_settle) >= SETTLE_STEP +} + /// Démarre le préchargement du clip suivant sur un thread dédié dès qu'on entre dans la /// fenêtre `PREFETCH_LEAD_SEC` avant la fin du clip actif — pour que la bascule à la /// frontière (`advance_to_next_scene_clip`) trouve les décodeurs déjà ouverts et positionnés @@ -1348,6 +1371,15 @@ unsafe fn render_thread( let mut last_preview_size: (u32, u32) = (0, 0); let mut last_ip: Option = None; let mut last_smoothing: f32 = -1.0; // force la 1re application (0.0 est une valeur valide) + // Fenêtre de stabilisation après un changement EN PAUSE. Un seul recompose ne suffit pas + // quand l'effet demandé est asynchrone : la segmentation webcam (détourage / flou / fond + // personnalisé) démarre son worker au 1er compose, ne SOUMET la frame qu'au 2e et ne + // téléverse le masque qu'au 3e — d'où l'effet qui n'apparaissait qu'au scrub suivant, le + // scrub étant la seule chose qui recomposait encore. + // ponytail: fenêtre fixe plutôt qu'un vrai signal « masque en attente » exposé par les + // trois compositeurs ; à remplacer si une machine met plus que ça à inférer. + let mut settle_until: Option = None; + let mut last_settle = Instant::now(); // La vue live est TOUJOURS pilotée par la scène de l'app. Tant qu'aucune scène n'a été // appliquée, on refuse de jouer le layout fixture (POC) : un fallback fixture ne ferait que // MASQUER un scene-push cassé. On attend la scène avant de produire le 1er frame. @@ -1575,6 +1607,11 @@ unsafe fn render_thread( if let Some(target) = requested { if player.present_frame(&comp, &cfg, target)? { stepped = true; + // Un seek en pause compose UNE fois, exactement comme un changement de param : + // la frame webcam a changé, donc son masque aussi, et il arrivera deux composes + // plus tard. Sans cette fenêtre, le masque de la position PRÉCÉDENTE reste + // affiché jusqu'à ce qu'une autre action provoque un compose. + (settle_until, last_settle) = open_settle_window(now); } acc = 0.0; // resynchronise l'accumulateur de lecture libre après un seek } else if shared.playing.load(Ordering::Relaxed) { @@ -1687,6 +1724,14 @@ unsafe fn render_thread( } } else if first || ip_changed || scene_changed || clip_changed || resized { // pause : recompose la frame courante (param / scène / clip / résolution changés). + (settle_until, last_settle) = open_settle_window(now); + let _ = player.recompose(&comp, &cfg); + stepped = true; + } else if should_settle(now, settle_until, last_settle) { + // Rien n'a changé, mais un masque de segmentation peut encore être en vol : on + // recompose à la cadence de la segmentation (pas à celle de la boucle) jusqu'à ce + // que la fenêtre expire. + last_settle = now; let _ = player.recompose(&comp, &cfg); stepped = true; } @@ -1905,6 +1950,44 @@ pub fn run_standalone(_screen: &str, _webcam: &str, _cursor_json: &str) -> anyho #[cfg(test)] mod tests { + use super::{open_settle_window, should_settle, SETTLE_STEP, SETTLE_WINDOW}; + use std::time::Instant; + + /// Le bug d'origine : en pause, un seul recompose par changement, donc le masque de + /// segmentation (asynchrone, 3 composes de latence) n'arrivait jamais avant un scrub. + #[test] + fn settle_recomposes_within_the_window_at_the_segmentation_rate() { + let t0 = Instant::now(); + let deadline = Some(t0 + SETTLE_WINDOW); + + assert!(!should_settle(t0, None, t0), "aucune fenêtre ouverte : rien à faire"); + assert!( + !should_settle(t0 + SETTLE_STEP / 2, deadline, t0), + "dans la fenêtre mais trop tôt : on ne recompose pas à la cadence de la boucle", + ); + assert!( + should_settle(t0 + SETTLE_STEP, deadline, t0), + "dans la fenêtre et la cadence est due : c'est le tour qui livre le masque", + ); + assert!( + !should_settle(t0 + SETTLE_WINDOW, deadline, t0), + "fenêtre expirée : on retombe en pause inerte plutôt que de recomposer sans fin", + ); + } + + /// Un seek en pause compose aussi UNE seule fois : la frame webcam a changé, son masque + /// arrive deux composes plus tard. Le chemin `present_frame` doit donc ouvrir la même + /// fenêtre que le chemin « un param a changé », sans recomposer immédiatement. + #[test] + fn a_paused_seek_opens_the_same_window() { + let t0 = Instant::now(); + let (until, last) = open_settle_window(t0); + + assert!(!should_settle(t0, until, last), "pas de recompose en boucle juste après le seek"); + assert!(should_settle(t0 + SETTLE_STEP, until, last), "le tour suivant livre le masque"); + assert!(!should_settle(t0 + SETTLE_WINDOW, until, last), "puis la fenêtre se referme"); + } + use super::*; fn multiclip_scene() -> Scene { diff --git a/crates/compositor/src/mac_frames.rs b/crates/compositor/src/mac_frames.rs index ec2432637..0d04b3da5 100644 --- a/crates/compositor/src/mac_frames.rs +++ b/crates/compositor/src/mac_frames.rs @@ -375,3 +375,51 @@ impl Drop for CpuFrames { } } } + +/// Fabrique un `CVPixelBufferRef` NV12 IOSurface-backed et y écrit les deux plans donnés. +/// +/// Réservé aux tests, mais posé ICI plutôt que dans le module de test : `CVPixelBufferCreate`, +/// le dictionnaire d'attributs IOSurface/Metal et le verrou d'accès CPU sont déjà écrits +/// au-dessus, et les redéclarer ailleurs ferait vivre deux copies de la même FFI. +/// +/// `y` fait `w * h` octets, `uv` fait `w * (h / 2)` (Cb, Cr entrelacés, demi-résolution). +/// C'est exactement ce que `CVMetalTextureCache` sait présenter en `R8Unorm` + `RG8Unorm`, +/// donc ce que `Compositor::nv12_srvs` attend — la même route qu'une frame VideoToolbox. +#[cfg(test)] +pub(crate) fn nv12_pixel_buffer_from_planes( + w: u32, + h: u32, + y: &[u8], + uv: &[u8], +) -> Result { + let (w, h) = (w as usize, h as usize); + if y.len() < w * h || uv.len() < w * (h / 2) { + bail!( + "plans trop courts pour {w}x{h} : Y={} octets, UV={} octets", + y.len(), + uv.len() + ); + } + unsafe { + let pb = create_nv12_pixel_buffer(w, h)?; + if CVPixelBufferLockBaseAddress(pb.as_ptr(), 0) != 0 { + bail!("CVPixelBufferLockBaseAddress (fixture NV12)"); + } + let base_y = CVPixelBufferGetBaseAddressOfPlane(pb.as_ptr(), 0); + let pitch_y = CVPixelBufferGetBytesPerRowOfPlane(pb.as_ptr(), 0); + let base_uv = CVPixelBufferGetBaseAddressOfPlane(pb.as_ptr(), 1); + let pitch_uv = CVPixelBufferGetBytesPerRowOfPlane(pb.as_ptr(), 1); + if base_y.is_null() || base_uv.is_null() { + CVPixelBufferUnlockBaseAddress(pb.as_ptr(), 0); + bail!("plans nuls (fixture NV12)"); + } + for row in 0..h { + ptr::copy_nonoverlapping(y.as_ptr().add(row * w), base_y.add(row * pitch_y), w); + } + for row in 0..h / 2 { + ptr::copy_nonoverlapping(uv.as_ptr().add(row * w), base_uv.add(row * pitch_uv), w); + } + CVPixelBufferUnlockBaseAddress(pb.as_ptr(), 0); + Ok(pb) + } +} diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 910738fc0..387e29319 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -14,6 +14,12 @@ //! (`timeline_walk::walk_composited_timeline`) et le muxer passe par le shim C //! `sn_fmt_set_pb` (comme Windows/macOS). **L'audio AAC n'est pas encore muxé** //! (increment suivant : `audio.rs` + `AacEncoder` sont déjà partagés). +//! +//! **Le débit demandé n'est pas un contrat sur ce chemin.** `libopenh264` n'a pas +//! de contrôle de débit utilisable — la cause est en amont, pas ici, et elle est +//! documentée avec ce qui a été mesuré dans `VideoEncoder::tune_openh264`. À lire +//! avant de toucher au calcul de `bit_rate` ou d'ajouter une option d'encodage : +//! la moitié des réglages qui semblent évidents ont été essayés et ne font rien. use anyhow::{bail, Result}; use std::collections::HashMap; @@ -21,9 +27,10 @@ use std::ffi::CString; use std::ptr; use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, + AacEncoder, PlanarPcm, }; +use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::config::Cfg; use crate::d3d::Gpu; use crate::ffi::AVFrame; @@ -31,10 +38,6 @@ use crate::linux_decode::SwDecoder; use crate::timeline_walk::NextFrameTime; use crate::linux_frames::CpuFrames; -/// `SWS_POINT` (plus proche voisin). Bindgen ne genere pas les `SWS_*` (macros), -/// valeur figee par l'ABI de libswscale -- comme `linux_frames::SWS_POINT`. -const SWS_POINT: i32 = 0x10; - /// Bilan d'un run d'export. Memes champs que `pipeline_macos::Stats`. pub struct Stats { pub frames: u64, @@ -97,6 +100,12 @@ pub struct Decoder { unsafe impl Send for Decoder {} impl Decoder { + /// Même point d'entrée que sur macOS, pour que `timeline_walk` reste portable. Le backend + /// Linux décode déjà en logiciel (`SwDecoder`) : l'intention n'a rien à trancher. + pub fn open_for_export(path: &str, gpu: &Gpu) -> Result { + Self::open(path, gpu) + } + pub fn open(path: &str, gpu: &Gpu) -> Result { let sw = SwDecoder::open(path)?; let fps = sw.fps(); @@ -194,10 +203,6 @@ impl Decoder { /// `pipeline_macos::VideoEncoder`. pub struct VideoEncoder { ctx: *mut crate::ffi::AVCodecContext, - /// AVFrame YUV420P envoyee a l'encodeur. - sw: *mut AVFrame, - /// RGBA (sortie compositeur) -> YUV420P. Cree paresseusement (dims du readback). - sws: *mut crate::ffi::SwsContext, w: i32, h: i32, } @@ -260,66 +265,154 @@ impl VideoEncoder { (*ctx).time_base = AVRational { num: 1, den: fps }; (*ctx).framerate = AVRational { num: fps, den: 1 }; (*ctx).bit_rate = bit_rate; + // Une image clé toutes les 2 s. SANS ce réglage le MP4 exporté n'en contient + // qu'UNE SEULE : le wrapper ffmpeg de `libopenh264` pose `g = -1` dans ses + // `FFCodecDefault`, et `try_open` ne touchait pas `gop_size`, donc openh264 + // recevait `uiIntraPeriod = 0` — mesuré 1 image I pour 300 frames. Le fichier + // reste lisible mais tout seek doit redécoder depuis le début, et un paquet + // abîmé emporte le reste de la vidéo. 2 s est le compromis usuel pour un + // fichier de sortie ; mesuré sur un vrai enregistrement 1080p60 il coûte + // +5,7 % de débit sur du contenu dense et +14,6 % sur un écran statique. + // Le défaut générique d'`AVCodecContext` (12 frames, soit 0,2 s à 60 fps) + // serait bien plus cher : on le pose donc explicitement pour tous les + // encodeurs, pas seulement pour celui qui a le défaut cassé. + (*ctx).gop_size = (fps * 2).max(1); + Self::tune_openh264(ctx, name); // MP4 : header global dans l'extradata (pas par-paquet). (*ctx).flags |= AV_CODEC_FLAG_GLOBAL_HEADER as i32; if let Err(e) = averr(avcodec_open2(ctx, enc, ptr::null_mut()), "avcodec_open2(enc)") { avcodec_free_context(&mut ctx); return Err(e); } - match alloc_sw_frame(AVPixelFormat::AV_PIX_FMT_YUV420P, w, h) { - Ok(sw) => Ok(VideoEncoder { ctx, sw, sws: ptr::null_mut(), w, h }), - Err(e) => { - avcodec_free_context(&mut ctx); - Err(e) - } - } + // Plus d'AVFrame ni de `SwsContext` ici : les frames viennent du pool de + // l'`EncodeWorker`, deja a la disposition du GPU. Cet encodeur ne + // possede plus que son contexte, donc il n'y a plus rien qui puisse + // echouer apres `avcodec_open2`. + Ok(VideoEncoder { ctx, w, h }) } - /// Envoie une frame composee DEJA RELUE (RGBA) a l'encodeur, en YUV420P. + /// Réglages propres à `libopenh264`, à poser AVANT `avcodec_open2` (openh264 fige + /// ses `SEncParamExt` à l'ouverture). No-op pour tout autre encodeur. + /// + /// **`libopenh264` n'a pas de contrôle de débit utilisable, et ce n'est pas + /// réparable ici** (issue #572). Le wrapper ffmpeg laisse `bEnableFrameSkip = 0`, + /// et openh264 le dit lui-même à l'ouverture : + /// + /// > `bEnableFrameSkip = 0, bitrate can't be controlled for RC_QUALITY_MODE,` + /// > `RC_BITRATE_MODE and RC_TIMESTAMP_MODE without enabling skip frame.` + /// + /// La seule option qui rétablit un vrai plafond est `allow_skip_frames`, et elle + /// le paie en frames jetées — mesuré 3 frames sur 120 conservées sur du contenu + /// incompressible. Inacceptable pour un export, donc on n'y touche pas. Sont + /// aussi des impasses vérifiées à la mesure : `rc_max_rate` et `rc_buffer_size` + /// (que ce wrapper ne lit pas, ou dont openh264 ne se sert que dans le chemin + /// frame-skip), `rc_mode`, `max_nal_size` et `level`. + /// + /// Restent deux réglages qui, eux, se mesurent : + /// + /// 1. **Ouvrir la fenêtre de QP.** Le wrapper ne transmet `iMinQp`/`iMaxQp` que + /// si `qmin`/`qmax` sont >= 0, et ses `FFCodecDefault` les posent à -1. + /// openh264 part alors sur ses propres défauts (0, 51) — puis sa + /// `ParamValidation()` juge `iMinQp = 0` invalide et REMPLACE toute la fenêtre + /// par (12, 42) (`GOM_MIN_QP_MODE`, `MAX_LOW_BR_QP`). Un `qmin` >= 1 évite la + /// substitution et rend le QP 51 atteignable. La borne basse, elle, reste + /// clampée à 12 quoi qu'on demande, donc `qmin = 1` ne fait qu'éviter le piège. + /// + /// L'intérêt premier est de ne plus subir une fenêtre qu'on n'a pas choisie : + /// (12, 42) n'est pas une décision, c'est ce qu'openh264 substitue en silence. + /// + /// Sur les quatre classes de contenu réel essayées, le changement est INERTE — + /// sortie identique à l'octet sur un écran statique, sur une capture dense, sur + /// du mixte (UI + fenêtre vidéo sur un tiers de l'image) et sur de la webcam + /// plein cadre. Sur ces mêmes clips le débit demandé est d'ailleurs plutôt bien + /// suivi (mixte 1080p30 : 1 -> 0,98, 2 -> 1,94, 4 -> 3,67 Mbps ; webcam : + /// 2 -> 1,99, 8 -> 7,51), ce qui vaut d'être su avant de conclure du titre de + /// #572 que l'export Linux serait à l'abandon. /// - /// La relecture est sortie d'ici : avec la ring de staging, la frame rendue - /// par `readback_submit` n'est pas celle qui vient d'etre composee mais la - /// precedente, donc l'appelant doit apparier lui-meme la frame et son pts - /// (cf. `run_composited_multi`). - pub unsafe fn send_rgba(&mut self, rgba: &[u8], rw: i32, rh: i32, pts: i64) -> Result<()> { + /// La fenêtre ne mord que sur du contenu que l'encodeur ne sait pas comprimer, + /// et là elle échange de la qualité contre de la taille : sur 120 frames de + /// bruit incompressible 720p30 à 2 Mbps demandés, 113,2 -> 65,5 Mbps mais + /// SSIM 0,949 -> 0,652. C'est le bon sens de l'échange pour un export (le + /// fichier restait 56x au-dessus de la cible), mais si on le regrettait, + /// `qmin = 12 ; qmax = 42` fige exactement le comportement d'avant tout en + /// gardant le réglage explicite. + /// 2. **Profil High.** Par défaut ce wrapper produit du Constrained Baseline en + /// CAVLC (`profile_idc = 66`, `entropy_coding_mode_flag = 0`). Le profil High + /// active CABAC. Mesuré à qualité égale sur un vrai enregistrement 1080p60 + /// (VMAF 96,2 dans les deux cas) : -3,8 % de débit sur du contenu dense, + /// -8,0 % sur un écran statique. + /// + /// Ce qu'il reste de cassé après ça, et qui ne se règle pas depuis l'application : + /// le débit demandé n'est qu'une entrée faible d'un modèle complexité -> QP, borné + /// à [12, 51]. Sur un écran statique l'encodeur se colle à QP 12 et ne dépense pas + /// plus, quel que soit le `bit_rate` (mesuré 0,68 Mbps pour 8 comme pour 40 Mbps + /// demandés) ; sur du contenu qu'il ne sait pas comprimer il dépasse la cible sans + /// borne. Le mode `SCREEN_CONTENT_REAL_TIME` d'openh264 — celui qui conviendrait à + /// un enregistreur d'écran — n'est atteignable par aucune option ffmpeg. + /// Suivi en amont : cisco/openh264#3259 (fermé sans correctif). + unsafe fn tune_openh264(ctx: *mut crate::ffi::AVCodecContext, name: &str) { use crate::ffi::*; - if self.sws.is_null() { - self.sws = sws_getContext( - rw, - rh, - AVPixelFormat::AV_PIX_FMT_RGBA, - self.w, - self.h, - AVPixelFormat::AV_PIX_FMT_YUV420P, - // POINT : le compositeur est dimensionne a la sortie -> pas de - // mise a l'echelle, donc echantillonnage exact (cf. mac_frames). - SWS_POINT, - ptr::null_mut(), - ptr::null_mut(), - ptr::null(), - ); - if self.sws.is_null() { - bail!("sws_getContext {rw}x{rh} RGBA -> {}x{} YUV420P", self.w, self.h); - } + if name != "libopenh264" { + return; } - averr(av_frame_make_writable(self.sw), "make_writable")?; - // RGBA est un plan unique : data[0] + stride rw*4, les autres nuls. - let src_data: [*const u8; 4] = [rgba.as_ptr(), ptr::null(), ptr::null(), ptr::null()]; - let src_stride: [i32; 4] = [rw * 4, 0, 0, 0]; - let converted = sws_scale( - self.sws, - src_data.as_ptr(), - src_stride.as_ptr(), - 0, - rh, - (*self.sw).data.as_ptr() as *const *mut u8, - (*self.sw).linesize.as_ptr(), - ); - if converted <= 0 { - bail!("sws_scale RGBA->YUV420P : {converted} lignes"); + (*ctx).qmin = 1; + (*ctx).qmax = 51; + // L'encodeur expose AUSSI `profile` en option privée, mais le wrapper lit + // `avctx->profile` quand la privée vaut `AV_PROFILE_UNKNOWN` (son défaut, + // -99). Les deux chemins produisent le même SPS — vérifié, `profile_idc` + // passe à 100 dans les deux cas — donc on prend le champ : il est typé, + // vérifié à la compilation, et n'a pas besoin d'une `CString` ni d'une + // branche d'erreur, contrairement à `av_opt_set` sur `priv_data`. + (*ctx).profile = AV_PROFILE_H264_HIGH as i32; + } + + /// Recopie le buffer relu dans une AVFrame du pool. Les deux ont la MEME + /// disposition (`alloc_padded_yuv_frame`), donc c'est un seul bloc contigu : + /// pas de reformatage, juste un transfert hors de la memoire mappee avant que + /// la ring ne recycle le slot. + /// + /// C'EST UNE COPIE, ET ELLE RESTE. La supprimer voudrait dire encoder + /// directement depuis le buffer de staging, donc le maintenir mappe pendant + /// que le worker travaille, a travers une frontiere de thread. Le gain est le + /// meme ~0,30 ms/frame que ce memcpy coute deja ; le prix serait un slot wgpu + /// dont la duree de vie depend de l'encodeur. Pas le bon echange tant que ce + /// n'est pas ce thread-ci le goulot. + pub unsafe fn copy_into( + dst_frame: *mut AVFrame, + planes: &[u8], + rw: i32, + rh: i32, + enc_w: i32, + enc_h: i32, + ) -> Result<()> { + // Les DEUX bornes comptent. La verification de taille seule laisserait + // passer un buffer assez gros mais de mauvaise geometrie : la disposition + // serait recalculee depuis rw/rh et l'image sortirait silencieusement + // decalee, bien plus difficile a diagnostiquer qu'un echec franc. + if rw != enc_w || rh != enc_h { + bail!("copy_into {rw}x{rh} != encodeur {enc_w}x{enc_h}"); } - (*self.sw).pts = pts; - averr(avcodec_send_frame(self.ctx, self.sw), "send_frame") + let lay = YuvLayout::for_size(rw, rh); + if planes.len() < lay.total { + bail!("plans YUV tronques : {} octets pour {}", planes.len(), lay.total); + } + // REND LA FRAME ECRIVABLE AVANT DE LA REECRIRE. `avcodec_send_frame` + // prend une reference sur le buffer ; un encodeur qui garde la frame — + // parce qu'il a du delai, ou parce que `OPENSCREEN_EXPORT_ENCODER` en a + // choisi un autre — la tiendrait encore quand le pool la recycle, et on + // ecrirait dans une image en cours d'encodage. + // + // J'avais retire cet appel en le jugeant inutile : avec `libopenh264` le + // refcount EST retombe a 1 au retour, mesure. Mais c'est une propriete de + // CET encodeur-la, pas du pool, et rien dans le code ne la maintenait. + // Ici l'appel est gratuit quand elle tient (refcount 1 = no-op) et + // correct quand elle ne tient pas. Le buffer ne porte pas + // `AV_BUFFER_FLAG_READONLY`, donc pas de branche recopie a redouter. + crate::ffi::averr(crate::ffi::av_frame_make_writable(dst_frame), "make_writable")?; + debug_assert_eq!((*dst_frame).linesize[0] as usize, lay.bpr_y); + debug_assert_eq!((*dst_frame).linesize[1] as usize, lay.bpr_uv); + std::ptr::copy_nonoverlapping(planes.as_ptr(), (*dst_frame).data[0], lay.total); + Ok(()) } /// Flush : une frame nulle finalise le bitstream de l'encodeur. @@ -335,51 +428,326 @@ impl Drop for VideoEncoder { fn drop(&mut self) { unsafe { crate::ffi::avcodec_free_context(&mut self.ctx); - if !self.sw.is_null() { - crate::ffi::av_frame_free(&mut self.sw); - } - if !self.sws.is_null() { - crate::ffi::sws_freeContext(self.sws); - } } } } -/// Alloue une AVFrame systeme au format demande. Symetrique de -/// `pipeline_macos::alloc_sw_frame`. -unsafe fn alloc_sw_frame(pix_fmt: crate::ffi::AVPixelFormat::Type, w: i32, h: i32) -> Result<*mut AVFrame> { + +/// Geometrie du buffer relu : strides alignes a 256 (ce que +/// `copy_texture_to_buffer` impose) et offsets des trois plans dans l'allocation +/// unique. Calculee a UN SEUL endroit, parce que le producteur (le compositeur) +/// et le consommateur (l'AVFrame du pool) doivent s'accorder a l'octet pres. +#[derive(Clone, Copy)] +struct YuvLayout { + bpr_y: usize, + bpr_uv: usize, + off_u: usize, + off_v: usize, + total: usize, +} + +impl YuvLayout { + /// DERIVE de `Compositor::yuv_layout_for`, jamais recalculee. Cette + /// arithmetique existait ici en double, et c'est precisement le genre de + /// duplication qui ne casse rien tant qu'elle est identique : le producteur + /// (le compositeur, qui remplit le buffer) et le consommateur (l'AVFrame du + /// pool) doivent s'accorder A L'OCTET, et un ecart ne donnerait pas une + /// panne mais une image decalee. + fn for_size(w: i32, h: i32) -> YuvLayout { + let (bpr_y, bpr_uv, off_u, total) = crate::compositor::Compositor::yuv_layout_for( + w.max(0) as u32, + h.max(0) as u32, + crate::compositor::YuvFormat::I420, + ); + let ch = (h.max(0) as u64).div_ceil(2); + let size_uv = u64::from(bpr_uv) * ch; + YuvLayout { + bpr_y: bpr_y as usize, + bpr_uv: bpr_uv as usize, + off_u: off_u as usize, + off_v: (off_u + size_uv) as usize, + total: total as usize, + } + } +} + +/// Alloue une AVFrame YUV420P dont les `linesize` sont EXACTEMENT les strides du +/// buffer relu, et dont les trois plans se suivent dans une seule allocation, +/// dans le meme ordre. +/// +/// POURQUOI PAS `av_frame_get_buffer`. Il choisit ses propres strides — 1920 et +/// 960 en 1080p — la ou le GPU impose 2048 et 1024. Recopier de l'un vers +/// l'autre demandait 3240 petits memcpy decales par frame (~0,67 ms) ; avec une +/// disposition identique des deux cotes, la meme donnee se recopie d'un seul +/// bloc contigu (~0,30 ms). libopenh264 lit `linesize[i]` et `data[i]` tels +/// quels et se moque qu'un plan soit sur-stride. +/// +/// LA FRAME RESTE REFCOMPTEE (`av_buffer_alloc`). Sans `buf[0]`, `av_frame_ref` +/// a l'interieur d'`avcodec_send_frame` prend la branche « donnee non +/// refcomptee » et REFAIT une allocation plus une copie complete — a l'interieur +/// de l'encodeur, donc precisement la ou on ne penserait pas a la chercher. +unsafe fn alloc_padded_yuv_frame(w: i32, h: i32) -> Result<*mut AVFrame> { + let lay = YuvLayout::for_size(w, h); let mut frame = crate::ffi::av_frame_alloc(); if frame.is_null() { - bail!("av_frame_alloc (encodeur)"); + bail!("av_frame_alloc (pool)"); } - (*frame).format = pix_fmt as i32; + (*frame).format = crate::ffi::AVPixelFormat::AV_PIX_FMT_YUV420P as i32; (*frame).width = w; (*frame).height = h; - if crate::ffi::av_frame_get_buffer(frame, 32) < 0 { + let buf = crate::ffi::av_buffer_alloc(lay.total); + if buf.is_null() { crate::ffi::av_frame_free(&mut frame); - bail!("av_frame_get_buffer {w}x{h} pix_fmt={pix_fmt}"); + bail!("av_buffer_alloc {} octets", lay.total); } + let base = (*buf).data; + (*frame).buf[0] = buf; + (*frame).data[0] = base; + (*frame).data[1] = base.add(lay.off_u); + (*frame).data[2] = base.add(lay.off_v); + (*frame).linesize[0] = lay.bpr_y as i32; + (*frame).linesize[1] = lay.bpr_uv as i32; + (*frame).linesize[2] = lay.bpr_uv as i32; Ok(frame) } -/// Draine les paquets de l'encodeur vers le muxer. Symetrique de -/// `pipeline_macos::drain_encoder`. -unsafe fn drain_encoder( - ectx: *mut crate::ffi::AVCodecContext, +/// Etat du muxer MP4, deplacable en bloc sur le thread d'encodage. +/// +/// POURQUOI UN SEUL TYPE PLUTOT QUE QUATRE VARIABLES. `av_interleaved_write_frame` +/// touche `octx`, la piste video `ostream` et le paquet de travail `opkt` ; et +/// `AacEncoder` garde un `*mut AVStream` qui pointe DANS la table de flux de +/// `octx` (audio.rs). Les separer laisserait un pointeur vers l'interieur d'un +/// objet possede par un autre thread. Ils partent donc ensemble, ou pas du tout. +struct Muxer { octx: *mut crate::ffi::AVFormatContext, + pb: *mut crate::ffi::AVIOContext, ostream: *mut crate::ffi::AVStream, opkt: *mut crate::ffi::AVPacket, -) -> Result<()> { - use crate::ffi::*; - loop { - let r = avcodec_receive_packet(ectx, opkt); - if r == AVERROR_EOF || r == AVERROR_EAGAIN { - return Ok(()); + aac: AacEncoder, +} + +// SAFETY : aucun de ces pointeurs n'a d'affinite de thread. Le muxer est DEPLACE +// vers le worker puis rendu au thread appelant par le `join` ; il n'est jamais +// partage, d'ou `Send` sans `Sync`. +unsafe impl Send for Muxer {} + +impl Muxer { + /// Draine les paquets de l'encodeur vers le fichier. Symetrique de + /// `pipeline_macos::drain_encoder`. + unsafe fn drain(&mut self, ectx: *mut crate::ffi::AVCodecContext) -> Result<()> { + use crate::ffi::*; + loop { + let r = avcodec_receive_packet(ectx, self.opkt); + if r == AVERROR_EOF || r == AVERROR_EAGAIN { + return Ok(()); + } + averr(r, "receive_packet")?; + av_packet_rescale_ts(self.opkt, (*ectx).time_base, (*self.ostream).time_base); + averr( + av_interleaved_write_frame(self.octx, self.opkt), + "interleaved_write_frame", + )?; + av_packet_unref(self.opkt); + } + } + + /// Ferme le conteneur. La liberation, elle, est dans `Drop` : un `?` entre + /// l'ouverture et ici ne doit pas fuir le contexte ni le fichier. + unsafe fn finish(&mut self) -> Result<()> { + crate::ffi::averr(crate::ffi::av_write_trailer(self.octx), "write_trailer") + } +} + +impl Drop for Muxer { + fn drop(&mut self) { + unsafe { + crate::ffi::avio_closep(&mut self.pb); + crate::ffi::avformat_free_context(self.octx); + crate::ffi::av_packet_free(&mut self.opkt); + } + } +} + +/// Une frame remplie, en route vers l'encodeur. +struct EncJob { + frame: *mut AVFrame, + pts: i64, +} +// SAFETY : la frame appartient au pool et n'est touchee que par UN thread a la +// fois — le passage par le canal est le transfert de propriete. +unsafe impl Send for EncJob {} + +/// Une frame vidée que le worker rend au pool. +struct FreeFrame(*mut AVFrame); +// SAFETY : idem `EncJob`, dans l'autre sens. +unsafe impl Send for FreeFrame {} + +/// Encodeur + muxer deportes sur leur propre thread. +/// +/// POURQUOI. L'export tenait sur UN thread : decodage, composition, relecture, +/// de-padding puis encodage a la queue leu leu, pendant que sept coeurs ne +/// faisaient rien. `avcodec_send_frame` pese a lui seul 29,5 s des ~57 s d'un +/// export de 3600 frames ; le sortir du chemin critique laisse la marche de +/// timeline avancer pendant que l'encodeur travaille la frame precedente. +/// +/// LE POOL BORNE LA MEMOIRE, PAS UN CANAL. Le thread de marche va plus vite que +/// l'encodeur : une file non bornee finirait par contenir les 3600 frames, soit +/// ~11,2 Go. Ici il existe EXACTEMENT `depth` AVFrames, qui tournent entre le +/// canal `empty` et le canal `full`. Le depassement n'est pas evite, il est +/// inexprimable — et `empty_rx.recv()` est le seul point ou la marche attend +/// l'encodeur, donc le seul endroit a instrumenter si le debit deçoit. +/// +/// LE DE-PADDING RESTE COTE MARCHE. Recopier les plans depuis le buffer relu +/// (lignes alignees a 256) vers l'AVFrame coute ~0,67 ms par frame. Le mettre +/// ici le poserait sur le thread qui est desormais le goulot ; le laisser sur la +/// marche, qui a du mou, ne coute rien. Meme raison pour laquelle il ne sert a +/// rien de donner la memoire mappee du GPU directement a l'encodeur : ca +/// supprimerait cette copie sans deplacer le goulot, en echange d'un slot de +/// staging maintenu mappe a travers une frontiere de thread. +struct EncodeWorker { + full_tx: Option>, + empty_rx: std::sync::mpsc::Receiver, + /// Frame empruntee mais finalement pas remplie — l'amorcage de la ring de + /// relecture ne produit rien les premiers tours — gardee ici pour le tour + /// suivant. `null` quand il n'y en a pas. + /// + /// POURQUOI PAS UN CLONE DU `Sender`. C'etait la premiere version, et elle + /// interdisait de detecter la mort du worker : tant que `EncodeWorker` + /// gardait un emetteur vivant, `empty_rx.recv()` ne pouvait JAMAIS rendre + /// `Err`, donc un worker qui panique laissait la marche bloquee pour + /// toujours sur `take_free` — `finish` n'etait jamais atteint. Le canal ne + /// doit avoir qu'un seul emetteur, celui du worker, pour que sa disparition + /// soit observable. + spare: std::cell::Cell<*mut AVFrame>, + handle: Option>>, + /// Premiere erreur rencontree par le worker. La marche la relit a chaque + /// frame : sans ca, un encodeur mort a la frame 12 laisserait composer les + /// 3588 suivantes avant que quiconque s'en apercoive. + fatal: std::sync::Arc>>, +} + +impl EncodeWorker { + /// Demarre le thread et alloue le pool. `enc` et `mux` lui appartiennent + /// jusqu'au `finish`. + fn spawn(mut enc: VideoEncoder, mut mux: Muxer, depth: usize) -> Result { + let (full_tx, full_rx) = std::sync::mpsc::channel::(); + let (empty_tx, empty_rx) = std::sync::mpsc::channel::(); + for _ in 0..depth.max(2) { + let f = unsafe { alloc_padded_yuv_frame(enc.w, enc.h)? }; + empty_tx + .send(FreeFrame(f)) + .map_err(|_| anyhow::anyhow!("pool d'encodage: canal ferme a l'amorcage"))?; + } + let fatal = std::sync::Arc::new(std::sync::Mutex::new(None::)); + let fatal_worker = std::sync::Arc::clone(&fatal); + let handle = std::thread::Builder::new() + .name("openscreen-encode".into()) + .spawn(move || -> Result { + while let Ok(job) = full_rx.recv() { + let r = unsafe { + (*job.frame).pts = job.pts; + crate::ffi::averr( + crate::ffi::avcodec_send_frame(enc.ctx, job.frame), + "send_frame", + ) + .and_then(|()| mux.drain(enc.ctx)) + }; + // La frame retourne au pool DANS TOUS LES CAS : la garder + // sur une erreur bloquerait la marche sur `empty_rx.recv()` + // au lieu de lui laisser voir `fatal`. + let _ = empty_tx.send(FreeFrame(job.frame)); + if let Err(e) = r { + *fatal_worker.lock().unwrap() = Some(format!("{e:#}")); + return Err(e); + } + } + // Canal ferme = plus aucune frame ne viendra : on vide + // l'encodeur ici, pendant qu'il nous appartient encore. + unsafe { + enc.flush()?; + mux.drain(enc.ctx)?; + } + Ok(mux) + })?; + Ok(EncodeWorker { + full_tx: Some(full_tx), + empty_rx, + spare: std::cell::Cell::new(std::ptr::null_mut()), + handle: Some(handle), + fatal, + }) + } + + /// Garde une frame empruntee sans avoir ete remplie, pour le tour suivant. + fn give_back(&self, frame: *mut AVFrame) { + let prev = self.spare.replace(frame); + debug_assert!(prev.is_null(), "give_back deux fois sans take_free"); + } + + /// Emprunte une frame libre au pool. C'est ICI que la marche attend quand + /// l'encodeur prend du retard. + fn take_free(&self) -> Result<*mut AVFrame> { + let spare = self.spare.replace(std::ptr::null_mut()); + if !spare.is_null() { + return Ok(spare); + } + match self.empty_rx.recv() { + Ok(FreeFrame(f)) => Ok(f), + Err(_) => Err(self.fatal_error("le thread d'encodage s'est arrete")), + } + } + + fn submit(&self, frame: *mut AVFrame, pts: i64) -> Result<()> { + match self.full_tx.as_ref() { + Some(tx) => tx + .send(EncJob { frame, pts }) + .map_err(|_| self.fatal_error("le thread d'encodage s'est arrete")), + None => Err(anyhow::anyhow!("submit apres finish")), + } + } + + /// Prefere l'erreur reelle du worker au symptome (« canal ferme »). + fn fatal_error(&self, fallback: &str) -> anyhow::Error { + match self.fatal.lock().unwrap().clone() { + Some(e) => anyhow::anyhow!("encodage: {e}"), + None => anyhow::anyhow!("{fallback}"), + } + } + + /// Ferme la file, attend le worker et RECUPERE le muxer : le `join` est + /// l'arete de synchronisation qui rend `octx` utilisable ici pour l'audio et + /// le trailer. + fn finish(&mut self) -> Result { + drop(self.full_tx.take()); + let handle = self + .handle + .take() + .ok_or_else(|| anyhow::anyhow!("finish appele deux fois"))?; + match handle.join() { + Ok(r) => r, + // Un panic du worker ne passe pas par `fatal` : le relayer en erreur + // plutot que de le repropager sur le thread de marche. + Err(_) => Err(self.fatal_error("le thread d'encodage a panique")), + } + } +} + +impl Drop for EncodeWorker { + fn drop(&mut self) { + // Chemin d'abandon (un `?` ailleurs) : fermer la file debloque le worker, + // et le join evite de liberer le pool sous ses pieds. + drop(self.full_tx.take()); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + let mut spare = self.spare.replace(std::ptr::null_mut()); + if !spare.is_null() { + unsafe { crate::ffi::av_frame_free(&mut spare) }; + } + while let Ok(FreeFrame(f)) = self.empty_rx.try_recv() { + let mut f = f; + unsafe { crate::ffi::av_frame_free(&mut f) }; } - averr(r, "receive_packet")?; - av_packet_rescale_ts(opkt, (*ectx).time_base, (*ostream).time_base); - averr(av_interleaved_write_frame(octx, opkt), "interleaved_write_frame")?; - av_packet_unref(opkt); } } @@ -390,6 +758,82 @@ unsafe fn drain_encoder( /// La marche de timeline est PARTAGEE (`walk_composited_timeline`) : elle compose /// chaque frame de sortie (vitesse/fenetrage/curseur inclus) puis appelle /// `on_frame(n)`, ou on relit + encode + draine. + +/// Fin de marche du chemin SOFTWARE : vider la ring de relecture, puis rendre au +/// compositeur sa profondeur par defaut. +/// +/// Le drain doit avoir lieu AVANT de fermer la file : les `depth - 1` dernieres +/// copies sont encore en vol, et sans lui la derniere frame composee ne serait +/// jamais encodee — video amputee d'une frame. +fn hw_none_tail( + comp: &crate::compositor::Compositor, + worker: &mut EncodeWorker, + out_w: u32, + out_h: u32, + encoded_pts: &mut i64, +) -> Result<()> { + unsafe { + loop { + let frame = worker.take_free()?; + let mut filled = false; + let got = comp.readback_take_yuv_with(|rw, rh, planes| { + VideoEncoder::copy_into(frame, planes, rw as i32, rh as i32, out_w as i32, out_h as i32)?; + filled = true; + Ok(()) + })?; + if filled { + worker.submit(frame, *encoded_pts)?; + *encoded_pts += 1; + } else { + worker.give_back(frame); + } + if !got { + break; + } + } + // Le compositeur peut survivre a l'export (l'appelant le possede) : on lui + // rend sa profondeur par defaut plutot que de lui laisser une ring a 2 et + // le buffer qui va avec. + comp.set_readback_yuv_depth(1)?; + } + Ok(()) +} + +/// Ou partent les frames composees. Voir le commentaire au point de choix. +enum Sink { + /// Encodage software, deporte sur un thread. + Software(Box), + /// Encodage materiel depuis un dmabuf, sur place. + /// + /// PLUSIEURS TAMPONS, PAS UN. Avec un seul, composer et encoder se + /// serialisent : le GPU compose, on l'attend, on encode, et rien ne se + /// recouvre. Deux tampons suffisent a decaler d'une frame — on compose la + /// n pendant que la n-1 s'encode — et c'est le meme raisonnement que la + /// profondeur 2 de la ring de relecture software. + Hardware { + enc: VaapiEncoder, + mux: Muxer, + staging: Vec, + /// Frame soumise mais pas encore encodee : (slot, soumission, pts). + pending: Option<(usize, wgpu::SubmissionIndex, i64)>, + /// Frame mappee remise a l'encodeur pour le slot precedent, gardee VIVANTE + /// tant qu'il peut la lire. `(slot, frame)`. + in_flight: Option<(usize, *mut AVFrame)>, + next: usize, + }, +} + +impl Sink { + /// Le worker software. Ne doit etre appele qu'apres avoir ecarte le cas + /// materiel — le chemin materiel n'en a pas. + fn worker(&mut self) -> &mut EncodeWorker { + match self { + Sink::Software(w) => w, + Sink::Hardware { .. } => unreachable!("worker() sur le chemin materiel"), + } + } +} + pub fn run_composited_multi( clips: &[ClipSource], out: &str, @@ -404,12 +848,89 @@ pub fn run_composited_multi( } let (out_w, out_h) = (params.width, params.height); let out_fps = params.fps.unwrap_or(30) as i32; - // bitrate proportionnel a la surface (reference : 8 Mbps @ 1920x1080). + // bitrate proportionnel a la surface (reference : 8 Mbps @ 1920x1080). Formule + // IDENTIQUE a celle de `pipeline_macos.rs` et `pipeline_windows.rs` : la garder + // alignee est ce qui fait que les trois plateformes exportent au meme poids. + // + // Sur `libopenh264` ce nombre n'est qu'indicatif : c'est une entree d'un modele + // complexite -> QP, pas un contrat. Il agit comme un plafond APPROXIMATIF sur du + // contenu dense (mesure sur un vrai enregistrement 1080p60 : 1 Mbps demande -> + // 0,97 produit, 2 -> 1,72, 4 -> 2,86, 8 -> 3,85) et n'a aucun effet sur un ecran + // statique, ou l'encodeur sature son plancher de QP. Voir + // `VideoEncoder::tune_openh264` pour le pourquoi et ce qui a ete tente. let bit_rate = ((out_w as i64 * out_h as i64 * 8_000_000) / (1920 * 1080)).max(2_000_000); let t0 = std::time::Instant::now(); - let mut enc = VideoEncoder::open(¶ms.codec, out_w as i32, out_h as i32, out_fps, bit_rate)?; - let ectx = enc.ctx; + // L'ENCODEUR SE CHOISIT AVANT LE MUXER, parce que c'est lui qui decrit le + // flux video. `h264_vaapi` s'il s'ouvre et que le compositeur sait exporter + // sa memoire ; sinon l'encodeur software, inchange. + // + // Le repli couvre plus que l'absence de GPU : pas de `/dev/dri/renderD128`, + // un pilote sans VAAPI, un device wgpu ouvert sans les extensions de memoire + // externe. Aucun de ces cas n'est une erreur — l'export doit juste rester + // celui d'avant. + // L'ECHAPPATOIRE DOIT AUSSI COUVRIR CE CHOIX. `OPENSCREEN_EXPORT_ENCODER` + // existe pour forcer un encodeur ; si le chemin materiel l'ignorait, demander + // `libopenh264` donnerait quand meme du VAAPI — et le reglage servirait + // surtout a diagnostiquer, donc mentir ici est pire qu'ailleurs. + let forced = std::env::var("OPENSCREEN_EXPORT_ENCODER").ok(); + let hw_allowed = match forced.as_deref() { + None => true, + Some(name) => name.contains("vaapi"), + }; + let hw = if hw_allowed && matches!(params.codec, ExportCodec::H264) { + unsafe { VaapiEncoder::open(out_w as i32, out_h as i32, out_fps, bit_rate) } + .and_then(|v| { + // PLUS DE TAMPONS QUE L'ENCODEUR N'A DE LATENCE. Deux suffisaient + // pour recouvrir composition et encodage, mais pas pour la + // question de propriete : `h264_vaapi` garde plusieurs frames + // avant d'emettre le premier paquet, donc a deux tampons on + // revenait sur le slot 0 alors que la surface qui le mappe etait + // encore detenue. Le garde-fou de `frame_released` le prouve — + // avec deux, il declenche des la premiere boucle. + // + // Six, pas deux : c'est au-dessus de la latence observee, ca + // coute 6 x 3,3 Mo, et le garde-fou reste en place pour le cas ou + // un pilote irait plus loin. + let total = comp.nv12_geometry().3; + let mut v_st = Vec::new(); + for _ in 0..6 { + v_st.push(comp.create_exportable_staging(total)?); + } + Some((v, v_st)) + }) + } else { + None + }; + // N'OUVRE PAS L'ENCODEUR SOFTWARE SI LE MATERIEL A GAGNE. Il etait construit + // dans tous les cas, donc alloue puis jamais utilise — visible par deux + // lignes « encodeur video » dans le log, et par un AVFrame de 3,1 Mo qui ne + // sert a rien. + let enc = match &hw { + Some(_) => None, + None => Some(VideoEncoder::open( + ¶ms.codec, + out_w as i32, + out_h as i32, + out_fps, + bit_rate, + )?), + }; + // ALIAS LU UNIQUEMENT AVANT LE DEMARRAGE DU WORKER. Il ne sert qu'a decrire + // le flux au muxer, juste en dessous ; passe `EncodeWorker::spawn`, le + // contexte appartient au thread d'encodage et cette variable ne doit plus + // etre touchee. S'en resservir apres serait un `Sync` officieux : + // `VideoEncoder` est `Send` et volontairement pas `Sync`, et un + // `*mut AVCodecContext` recopie efface exactement cette distinction. + let ectx = match (&hw, &enc) { + (Some((v, _)), _) => v.ctx(), + (None, Some(e)) => e.ctx, + (None, None) => bail!("aucun encodeur video disponible"), + }; + eprintln!( + "[pipeline] encodeur video : {}", + if hw.is_some() { "h264_vaapi (materiel, dmabuf)" } else { "software" } + ); let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); @@ -420,7 +941,7 @@ pub fn run_composited_multi( let mut pb: *mut crate::ffi::AVIOContext = ptr::null_mut(); let ostream; let opkt; - let mut audio_encoder; + let audio_encoder; unsafe { crate::ffi::averr( crate::ffi::avformat_alloc_output_context2(&mut octx, ptr::null(), ptr::null(), outc.as_ptr()), @@ -450,18 +971,47 @@ pub fn run_composited_multi( )?; opkt = crate::ffi::av_packet_alloc(); } + // A partir d'ici le muxer est un seul objet, et il part avec l'encodeur. + let mux = Muxer { octx, pb, ostream, opkt, aac: audio_encoder }; + // Profondeur 3 : deux frames en vol suffisent a couvrir l'encodeur, la + // troisieme absorbe les a-coups de la marche (une fin de clip y decode tout + // l'audio du clip d'un coup, cf. `on_clip_end`). + // Deux formes, pas deux variantes d'une meme : le chemin software encode sur + // un thread (l'encodeur y coute ~8 ms/frame, il faut le sortir du chemin + // critique), le chemin materiel encode sur place (~3 ms) et garde le muxer + // sous la main. Les melanger rendrait les deux illisibles. + let mut sink = match hw { + Some((venc, staging)) => Sink::Hardware { + enc: venc, + mux, + staging, + pending: None, + in_flight: None, + next: 0, + }, + None => { + let enc = enc.ok_or_else(|| anyhow::anyhow!("aucun encodeur video disponible"))?; + Sink::Software(Box::new(EncodeWorker::spawn(enc, mux, 3)?)) + } + }; // Un PCM par clip, assemble apres la marche video (elle seule dit combien de // frames chaque clip a produit, donc combien d'audio lui revient). - let mut clip_pcm: Vec> = (0..clips.len()).map(|_| None).collect(); + let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let mut clip_frame_counts: Vec = vec![0; clips.len()]; let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene so the + // mix step below owns them. Empty for a project with no imported audio. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour // la raison pour laquelle la preview, elle, reste a 1. - comp.set_readback_depth(2)?; + comp.set_readback_yuv_depth(2)?; // pts d'encodage : DECOUPLE de l'index de marche `n`, puisque la frame // recoltee a l'iteration n est celle composee a n-1. Il reste contigu (les // frames sortent de la ring dans l'ordre de composition), donc le fichier @@ -479,13 +1029,80 @@ pub fn run_composited_multi( &mut webcam_decs, &mut |n| { // Soumet la copie de la frame n SANS l'attendre et recolte la - // precedente : c'est tout le pipelining. Pendant que le CPU - // passe ses ~12,6 ms dans sws_scale + avcodec_send_frame sur la - // frame n-1, le GPU finit la composition et la copie de n. - if let Some((rw, rh, rgba)) = comp.readback_submit()? { - enc.send_rgba(&rgba, rw as i32, rh as i32, encoded_pts)?; + // precedente : c'est tout le pipelining GPU. L'encodage, lui, + // n'est plus ici du tout — il tourne sur `worker` pendant que + // cette closure compose deja la frame suivante. + match &mut sink { + Sink::Hardware { enc, mux, staging, pending, in_flight, next } => { + // Soumet la frame n SANS l'attendre, puis encode la + // precedente : le GPU compose pendant que l'encodeur + // travaille. La toute premiere passe n'a rien a encoder, + // comme l'amorcage de la ring software. + let slot = *next; + // AVANT d'ecrire dans ce slot : s'assurer que l'encodeur + // ne lit plus la surface qui le mappait. Draine tant qu'il + // la retient — c'est le drain qui fait sortir les paquets + // et relache les references, donc la boucle progresse. + if let Some((busy, frame)) = in_flight.take() { + if busy == slot { + let mut spins = 0; + while !VaapiEncoder::frame_released(frame) { + mux.drain(enc.ctx())?; + spins += 1; + if spins > 1000 { + bail!("l'encodeur retient la surface du slot {slot}"); + } + } + let mut f = frame; + crate::ffi::av_frame_free(&mut f); + } else { + *in_flight = Some((busy, frame)); + } + } + let idx = comp.compose_into_dmabuf(&staging[slot])?; + if let Some((prev, prev_idx, pts)) = pending.take() { + comp.wait_submission(prev_idx); + let (bpr_y, bpr_uv, off_uv, _) = comp.nv12_geometry(); + let f = enc.send_dmabuf(staging[prev].fd, bpr_y, bpr_uv, off_uv, pts)?; + mux.drain(enc.ctx())?; + // Remplace le precedent : il a ete relache plus haut + // si son slot revenait, sinon il l'est par ce drain. + if let Some((_, old)) = in_flight.take() { + let mut o = old; + crate::ffi::av_frame_free(&mut o); + } + *in_flight = Some((prev, f)); + } + *pending = Some((slot, idx, encoded_pts)); + encoded_pts += 1; + *next = (slot + 1) % staging.len(); + progress(n + 1); + return Ok(()); + } + Sink::Software(_) => {} + } + let worker = sink.worker(); + let frame = worker.take_free()?; + let mut filled = false; + comp.readback_submit_yuv(|rw, rh, planes| { + VideoEncoder::copy_into( + frame, + planes, + rw as i32, + rh as i32, + out_w as i32, + out_h as i32, + )?; + filled = true; + Ok(()) + })?; + if filled { + worker.submit(frame, encoded_pts)?; encoded_pts += 1; - drain_encoder(ectx, octx, ostream, opkt)?; + } else { + // Amorcage de la ring : rien a encoder, la frame empruntee + // retourne au pool telle quelle. + worker.give_back(frame); } // Progression = frames COMPOSEES (inchangee) : la barre ne doit // pas reculer d'une frame parce que l'encodage a un tour de @@ -497,52 +1114,95 @@ pub fn run_composited_multi( clip_frame_counts[clip_index] = frames_in_clip; let clip = &clips[clip_index]; if clip.has_audio && frames_in_clip > 0 { - match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { - Ok(Some(pcm)) => { - clip_pcm[clip_index] = - Some(stretch_clip_pcm_by_speed(&pcm, speed_segments, out_fps as f64)); - } - Ok(None) => eprintln!( - "[pipeline] warning: clip #{clip_index} declare audio mais sans flux decodable; silence", - ), - Err(error) => eprintln!( - "[pipeline] warning: decodage audio clip #{clip_index} echoue ({error:#}); silence", - ), - } + // L'audio d'un clip ne dépend que de ce clip : le décoder et l'étirer ici, + // sur le thread de rendu, immobilisait la barre d'export pour toute sa + // durée — rien n'appelle `progress()` entre deux clips. Le travail part + // sur un thread et se recouvre avec la composition du clip suivant ; les + // résultats sont récupérés après le parcours, rangés par index de clip. + let path = clip.screen.clone(); + let source_start_sec = clip.source_start_sec; + let segments = speed_segments.to_vec(); + audio_jobs.spawn(clip_index, move || { + decode_and_stretch_clip_audio( + clip_index, + &path, + source_start_sec, + source_end_sec, + &segments, + out_fps as f64, + ) + }); } Ok(()) }, )? }; + // Le chemin materiel n'a ni ring ni file : il ne reste qu'a vider l'encodeur. + if let Sink::Hardware { enc, mux, staging, pending, in_flight, .. } = &mut sink { + unsafe { + // La derniere frame composee est encore en vol : sans ca la video + // sortirait amputee d'une frame, exactement comme le drain de la + // ring cote software. + if let Some((prev, prev_idx, pts)) = pending.take() { + comp.wait_submission(prev_idx); + let (bpr_y, bpr_uv, off_uv, _) = comp.nv12_geometry(); + let f = enc.send_dmabuf(staging[prev].fd, bpr_y, bpr_uv, off_uv, pts)?; + mux.drain(enc.ctx())?; + if let Some((_, old)) = in_flight.take() { + let mut o = old; + crate::ffi::av_frame_free(&mut o); + } + *in_flight = Some((prev, f)); + } + crate::ffi::avcodec_send_frame(enc.ctx(), ptr::null_mut()); + mux.drain(enc.ctx())?; + // Le flush a fait sortir tout ce qui restait : plus rien ne reference + // les surfaces, on peut liberer la derniere. + if let Some((_, f)) = in_flight.take() { + let mut f = f; + crate::ffi::av_frame_free(&mut f); + } + // Le compositeur survit a l'export : lui rendre sa profondeur par + // defaut vaut pour LES DEUX chemins. Le chemin materiel n'utilise pas + // la ring, mais `ensure_yuv_fmt` a pu la vider et la redimensionner, + // et la preview qui suit n'a pas a heriter de cet etat. + comp.set_readback_yuv_depth(1)?; + } + } + let mut mux = match sink { + Sink::Hardware { mux, .. } => mux, + Sink::Software(worker) => { + let mut worker = worker; + hw_none_tail(comp, &mut worker, out_w, out_h, &mut encoded_pts)?; + worker.finish()? + } + }; + unsafe { - // Drain de la ring AVANT le flush de l'encodeur : les `depth - 1` - // dernieres copies sont encore en vol, et sans ce drain la derniere - // frame composee ne serait jamais encodee (video amputee d'une frame). - while let Some((rw, rh, rgba)) = comp.readback_take()? { - enc.send_rgba(&rgba, rw as i32, rh as i32, encoded_pts)?; - encoded_pts += 1; - drain_encoder(ectx, octx, ostream, opkt)?; - } - // Le compositeur peut survivre a l'export (l'appelant le possede) : on - // lui rend sa profondeur par defaut plutot que de lui laisser une ring - // a 2 et le buffer de 8 Mo qui va avec. - comp.set_readback_depth(1)?; - enc.flush()?; - drain_encoder(ectx, octx, ostream, opkt)?; // Audio : le plan part des frames REELLEMENT produites par clip (un clip // raccourci voit son audio raccourci d'autant), puis un seul encode AAC. + // Récupération des jobs audio lancés pendant le parcours. `spawn` en admet quatre + // avant d'en collecter un, donc il en reste au plus quatre à attendre ici — bornés + // par le plus lent, pas par leur somme ; les autres se sont recouverts avec + // l'encodage vidéo. + let clip_pcm: Vec> = audio_jobs + .into_results() + .into_iter() + .map(|slot| slot.flatten()) + .collect(); + let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); - audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + let octx = mux.octx; + mux.aac.encode( + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; - crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; - crate::ffi::avio_closep(&mut pb); - crate::ffi::avformat_free_context(octx); - let mut opkt = opkt; - crate::ffi::av_packet_free(&mut opkt); + mux.finish()?; } let wall_s = t0.elapsed().as_secs_f64(); @@ -553,3 +1213,482 @@ pub fn run_composited_multi( video_duration_s: frames as f64 / out_fps as f64, }) } + +// --------------------------------------------------------------------------- +// Encodage materiel depuis un dmabuf +// --------------------------------------------------------------------------- + +/// Encodeur `h264_vaapi` alimente par un dmabuf, sans relecture CPU. +/// +/// POURQUOI `av_hwframe_map` ET JAMAIS `av_hwframe_transfer_data`. Le second +/// est le chemin d'UPLOAD CPU -> GPU, et c'est lui qui appelle `vaMapBuffer2`, +/// absent de libva avant 2.22 : sur Ubuntu 24.04 (libva 2.20) il ne rend pas une +/// erreur, il `assert(0)` et le processus meurt (cf. issue #552). Le mapping, +/// lui, ne prend pas ce chemin -- c'est ce qui rend cet encodeur utilisable la +/// ou l'upload ne l'est pas. +pub struct VaapiEncoder { + ctx: *mut crate::ffi::AVCodecContext, + drm_device: *mut crate::ffi::AVBufferRef, + va_device: *mut crate::ffi::AVBufferRef, + drm_frames: *mut crate::ffi::AVBufferRef, + va_frames: *mut crate::ffi::AVBufferRef, + w: i32, + h: i32, +} + +// SAFETY : memes raisons que `VideoEncoder` -- pointeurs FFI sans affinite de +// thread, un seul thread a la fois. +unsafe impl Send for VaapiEncoder {} + +/// Libere le descripteur porte par l'`AVBufferRef` de la frame source. +unsafe extern "C" fn drm_desc_free(_opaque: *mut std::ffi::c_void, data: *mut u8) { + crate::ffi::av_free(data as *mut std::ffi::c_void); +} + +impl VaapiEncoder { + /// Ouvre la chaine DRM -> VAAPI -> `h264_vaapi`. `None` si quoi que ce soit + /// manque : l'appelant retombe alors sur l'encodeur software. + pub unsafe fn open(w: i32, h: i32, fps: i32, bit_rate: i64) -> Option { + use crate::ffi::*; + let mut me = VaapiEncoder { + ctx: ptr::null_mut(), + drm_device: ptr::null_mut(), + va_device: ptr::null_mut(), + drm_frames: ptr::null_mut(), + va_frames: ptr::null_mut(), + w, + h, + }; + // LE DEVICE DRM D'ABORD, PUIS VAAPI DERIVE DE LUI. L'ordre inverse + // (VAAPI ouvert seul) rend ENOSYS sur radeonsi : le mapping veut les deux + // cotes d'un meme device. + let node = std::ffi::CString::new("/dev/dri/renderD128").ok()?; + if av_hwdevice_ctx_create( + &mut me.drm_device, + AVHWDeviceType::AV_HWDEVICE_TYPE_DRM, + node.as_ptr(), + ptr::null_mut(), + 0, + ) < 0 + { + return None; + } + if av_hwdevice_ctx_create_derived( + &mut me.va_device, + AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI, + me.drm_device, + 0, + ) < 0 + { + return None; + } + // `initial_pool_size = 0` sur LES DEUX contextes : ils ne font + // qu'ENVELOPPER des surfaces fournies de l'exterieur (le dmabuf d'un + // cote, ce que `av_hwframe_map` remplit de l'autre). Demander un pool + // pre-alloue fait rejeter le format par `av_hwframe_ctx_init` en EINVAL, + // faute d'allocateur pour ces dispositions. + let mk_frames = |dev: *mut AVBufferRef, fmt: AVPixelFormat::Type| -> *mut AVBufferRef { + let frames = av_hwframe_ctx_alloc(dev); + if frames.is_null() { + return ptr::null_mut(); + } + let c = (*frames).data as *mut AVHWFramesContext; + (*c).format = fmt; + (*c).sw_format = AVPixelFormat::AV_PIX_FMT_NV12; + (*c).width = w; + (*c).height = h; + (*c).initial_pool_size = 0; + if av_hwframe_ctx_init(frames) < 0 { + return ptr::null_mut(); + } + frames + }; + me.drm_frames = mk_frames(me.drm_device, AVPixelFormat::AV_PIX_FMT_DRM_PRIME); + me.va_frames = mk_frames(me.va_device, AVPixelFormat::AV_PIX_FMT_VAAPI); + if me.drm_frames.is_null() || me.va_frames.is_null() { + return None; + } + + let name = std::ffi::CString::new("h264_vaapi").ok()?; + let enc = avcodec_find_encoder_by_name(name.as_ptr()); + if enc.is_null() { + return None; + } + me.ctx = avcodec_alloc_context3(enc); + if me.ctx.is_null() { + return None; + } + (*me.ctx).width = w; + (*me.ctx).height = h; + (*me.ctx).pix_fmt = AVPixelFormat::AV_PIX_FMT_VAAPI as i32; + (*me.ctx).time_base = AVRational { num: 1, den: fps }; + (*me.ctx).framerate = AVRational { num: fps, den: 1 }; + (*me.ctx).bit_rate = bit_rate; + // MEME INTERVALLE D'IMAGES CLES QUE LE CHEMIN SOFTWARE, et pour la meme + // raison : deux secondes est le compromis choisi pour un fichier de + // sortie, il n'a pas a dependre de l'encodeur qui se trouve disponible. + // + // Sans cette ligne le resultat est correct A 60 FPS ET NULLE PART + // AILLEURS : `h264_vaapi` a un defaut de 120 frames (mesure), qui vaut + // deux secondes a 60 fps par coincidence. A 30 fps ca donnerait quatre + // secondes, a 120 fps une seule. Le defaut de `libopenh264` est -1, ce + // qui est un autre probleme encore (une seule image cle pour tout le + // fichier) traite dans `try_open`. + (*me.ctx).gop_size = (fps * 2).max(1); + // MP4 veut SPS/PPS dans l'extradata, pas repetes devant chaque image + // cle. Le chemin software le pose depuis toujours (`try_open`) ; l'avoir + // oublie ici produisait un fichier qui se lit quand meme, parce que le + // muxer recupere ce qu'il trouve — mais un lecteur qui se fie a + // `codecpar` seul aurait de quoi echouer. + (*me.ctx).flags |= AV_CODEC_FLAG_GLOBAL_HEADER as i32; + (*me.ctx).hw_frames_ctx = av_buffer_ref(me.va_frames); + if avcodec_open2(me.ctx, enc, ptr::null_mut()) < 0 { + return None; + } + Some(me) + } + + /// Envoie a l'encodeur l'image qui se trouve derriere `fd`, decrite comme un + /// NV12 lineaire de pitches `bpr_y` / `bpr_uv`. + pub unsafe fn send_dmabuf( + &mut self, + fd: i32, + bpr_y: u32, + bpr_uv: u32, + off_uv: u64, + pts: i64, + ) -> Result<*mut AVFrame> { + use crate::ffi::*; + let desc = av_mallocz(std::mem::size_of::()) + as *mut AVDRMFrameDescriptor; + if desc.is_null() { + bail!("av_mallocz(AVDRMFrameDescriptor)"); + } + (*desc).nb_objects = 1; + (*desc).objects[0].fd = fd; + // 0 : la taille est retrouvee par le pilote depuis le fd lui-meme. + (*desc).objects[0].size = 0; + (*desc).objects[0].format_modifier = 0; // DRM_FORMAT_MOD_LINEAR + (*desc).nb_layers = 1; + // fourcc 'NV12', ecrit a la main : bindgen ne genere pas MKTAG. + (*desc).layers[0].format = u32::from_le_bytes(*b"NV12"); + (*desc).layers[0].nb_planes = 2; + (*desc).layers[0].planes[0].object_index = 0; + (*desc).layers[0].planes[0].offset = 0; + (*desc).layers[0].planes[0].pitch = bpr_y as isize; + (*desc).layers[0].planes[1].object_index = 0; + (*desc).layers[0].planes[1].offset = off_uv as isize; + (*desc).layers[0].planes[1].pitch = bpr_uv as isize; + + let src = av_frame_alloc(); + (*src).format = AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + (*src).width = self.w; + (*src).height = self.h; + (*src).data[0] = desc as *mut u8; + // LA SOURCE DOIT ETRE REFCOMPTEE. Sans `buf[0]`, `av_hwframe_map` rend + // EINVAL -- et son message ne dit pas un mot de comptage de references, + // ce qui rend la panne tres difficile a lire. + (*src).buf[0] = av_buffer_create( + desc as *mut u8, + std::mem::size_of::(), + Some(drm_desc_free), + ptr::null_mut(), + 0, + ); + (*src).hw_frames_ctx = av_buffer_ref(self.drm_frames); + + let dst = av_frame_alloc(); + (*dst).format = AVPixelFormat::AV_PIX_FMT_VAAPI as i32; + (*dst).width = self.w; + (*dst).height = self.h; + (*dst).hw_frames_ctx = av_buffer_ref(self.va_frames); + // Bindgen range les `AV_HWFRAME_MAP_*` dans un module anonyme : les + // nommer par leur valeur serait plus fragile que de passer par lui. + let flags = (crate::ffi::_bindgen_ty_3::AV_HWFRAME_MAP_READ + | crate::ffi::_bindgen_ty_3::AV_HWFRAME_MAP_DIRECT) as i32; + let mapped = av_hwframe_map(dst, src, flags); + if mapped < 0 { + let mut s = src; + let mut d = dst; + av_frame_free(&mut s); + av_frame_free(&mut d); + averr(mapped, "av_hwframe_map(DRM -> VAAPI)")?; + unreachable!("averr rend une erreur pour mapped < 0"); + } + (*dst).pts = pts; + let r = averr(avcodec_send_frame(self.ctx, dst), "send_frame(vaapi)"); + // `src` a fini son role : `av_hwframe_map` a copie ce qu'il fallait dans + // `dst`, et le descripteur DRM meurt avec lui. + let mut s = src; + av_frame_free(&mut s); + // `dst` PAS libere ici. `avcodec_send_frame` en a pris une reference, et + // cette frame mappe le dmabuf du slot : tant qu'elle vit, l'encodeur peut + // encore lire cette memoire. L'appelant la garde et ne la relache — donc + // ne recycle le slot — qu'apres avoir draine le paquet correspondant. + r.map(|()| dst) + } + + /// Vrai si l'encodeur ne detient plus la frame mappee, donc si le slot qu'elle + /// couvre peut etre reecrit. + /// + /// C'est la SEULE question qui compte pour reutiliser un slot. Un `drain` qui + /// rend `EAGAIN` ne dit rien la-dessus : il signale qu'aucun paquet n'est + /// pret, pas que la surface est relachee. + pub unsafe fn frame_released(frame: *mut AVFrame) -> bool { + frame.is_null() + || (*frame).buf[0].is_null() + || crate::ffi::av_buffer_get_ref_count((*frame).buf[0]) <= 1 + } + + pub fn ctx(&self) -> *mut crate::ffi::AVCodecContext { + self.ctx + } +} + +impl Drop for VaapiEncoder { + fn drop(&mut self) { + unsafe { + if !self.ctx.is_null() { + crate::ffi::avcodec_free_context(&mut self.ctx); + } + for b in [ + &mut self.va_frames, + &mut self.drm_frames, + &mut self.va_device, + &mut self.drm_device, + ] { + if !b.is_null() { + crate::ffi::av_buffer_unref(b); + } + } + } + } +} + +#[cfg(test)] +mod vaapi_tests { + use super::*; + + /// La chaine complete, dans le crate et non dans un bac a sable : un tampon + /// de staging EXPORTABLE alloue par le compositeur, son fd donne a + /// `av_hwframe_map`, et `h264_vaapi` qui en sort un paquet. + /// + /// C'est le premier test qui touche reellement l'encodeur materiel. Il se + /// saute proprement partout ou la chaine n'existe pas (pas de GPU, pas de + /// `/dev/dri/renderD128`, pas de VAAPI) -- la CI rend sur lavapipe, et + /// l'echec y serait un faux negatif. + #[test] + fn vaapi_encodes_from_an_exported_dmabuf() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas d'adaptateur Vulkan — test saute"); + return; + }; + let (w, h) = (640i32, 480i32); + let comp = match crate::compositor::Compositor::new_sized(&gpu, w as u32, h as u32) { + Ok(c) => c, + Err(e) => { + eprintln!("compositeur indisponible ({e:#}) — test saute"); + return; + } + }; + let (bpr_y, bpr_uv, off_uv, total) = crate::compositor::Compositor::yuv_layout_for( + w as u32, + h as u32, + crate::compositor::YuvFormat::Nv12, + ); + let Some(st) = comp.create_exportable_staging(total) else { + eprintln!("pas de memoire externe — test saute"); + return; + }; + + // Du gris legal plutot que des zeros : un plan Y a 0 est du noir hors + // plage en BT.601 limite, et on veut que l'encodeur voie une image + // valide, pas qu'il la rattrape. + let mut grey = vec![128u8; total as usize]; + grey[..off_uv as usize].fill(128); + gpu.context.write_buffer(st.buffer(), 0, &grey); + gpu.context.submit(std::iter::empty()); + gpu.device.poll(wgpu::Maintain::Wait); + + unsafe { + let Some(mut enc) = VaapiEncoder::open(w, h, 60, 4_000_000) else { + eprintln!("h264_vaapi indisponible — test saute"); + return; + }; + enc.send_dmabuf(st.fd, bpr_y, bpr_uv, off_uv, 0) + .expect("send_dmabuf"); + // Un encodeur peut legitimement retenir la premiere frame : on le + // vide pour forcer la sortie du paquet. + let _ = crate::ffi::avcodec_send_frame(enc.ctx(), std::ptr::null_mut()); + let pkt = crate::ffi::av_packet_alloc(); + let r = crate::ffi::avcodec_receive_packet(enc.ctx(), pkt); + assert!(r >= 0, "avcodec_receive_packet a rendu {r}"); + assert!((*pkt).size > 0, "paquet H.264 vide"); + let mut p = pkt; + crate::ffi::av_packet_free(&mut p); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const W: i32 = 320; + const H: i32 = 180; + + /// Sortie d'un encodage de test : les drapeaux « image clé » paquet par paquet, + /// et l'extradata (le SPS, en Annex-B, puisque `try_open` pose + /// `AV_CODEC_FLAG_GLOBAL_HEADER`). + struct Encoded { + keyframes: Vec, + extradata: Vec, + qmin: i32, + qmax: i32, + gop_size: i32, + } + + /// Encode `n` images 320x180 par le chemin RÉEL de l'export — `VideoEncoder::open`, + /// donc `try_open`, donc toute la configuration que ce fichier pose. Un contexte + /// monté à la main dans le test ne prouverait rien. + /// + /// Rend `None` quand l'encodeur n'est pas ouvrable (build ffmpeg sans + /// `libopenh264`), pour que le test se saute proprement au lieu d'échouer sur + /// l'environnement. + fn encode(fps: i32, n: usize) -> Option { + // Un dev qui force un autre encodeur ne doit pas voir ce test rougir : il ne + // décrit que `libopenh264`. + match std::env::var("OPENSCREEN_EXPORT_ENCODER") { + Ok(name) if name != "libopenh264" => return None, + _ => {} + } + let mut enc = VideoEncoder::open(&ExportCodec::H264, W, H, fps, 1_000_000).ok()?; + + let mut out = Encoded { + keyframes: Vec::new(), + extradata: Vec::new(), + qmin: 0, + qmax: 0, + gop_size: 0, + }; + unsafe { + use crate::ffi::*; + out.qmin = (*enc.ctx).qmin; + out.qmax = (*enc.ctx).qmax; + out.gop_size = (*enc.ctx).gop_size; + let (ptr, len) = ((*enc.ctx).extradata, (*enc.ctx).extradata_size); + if !ptr.is_null() && len > 0 { + out.extradata = std::slice::from_raw_parts(ptr, len as usize).to_vec(); + } + + let pkt = av_packet_alloc(); + // Alimente par le CHEMIN DE PRODUCTION : un buffer a la disposition + // du GPU (`YuvLayout`) recopie par `copy_into`, exactement ce que + // fait l'export. Le test passait par `send_rgba`, qui n'a plus aucun + // appelant en production depuis que la conversion YUV est sur le GPU + // — il verifiait donc la configuration de l'encodeur en empruntant un + // chemin que plus personne ne prend. + let lay = YuvLayout::for_size(W, H); + let mut planes = vec![0u8; lay.total]; + let frame = alloc_padded_yuv_frame(W, H).expect("alloc_padded_yuv_frame"); + // Copie du pointeur AVANT la closure : la capturer via `enc.ctx` + // emprunterait `enc` en lecture pour toute la durée du drain, et + // `send_rgba` en veut un emprunt mutable juste après. + let ectx = enc.ctx; + let mut drain = |pkt: *mut AVPacket, out: &mut Encoded| loop { + let r = avcodec_receive_packet(ectx, pkt); + if r == AVERROR_EOF || r == AVERROR_EAGAIN { + return; + } + out.keyframes.push((*pkt).flags & AV_PKT_FLAG_KEY as i32 != 0); + av_packet_unref(pkt); + }; + for i in 0..n { + // Un motif qui glisse : de quoi donner du residu a coder, sans + // quoi l'encodeur travaille sur une image morte et le test ne + // dirait rien des images cles. + // Un degrade GROSSIER et de faible amplitude, qui glisse d'un pas + // par frame. Assez de residu pour que l'encodeur travaille, pas + // assez de rupture pour reveiller sa detection de changement de + // scene — un motif contraste donnait des images cles a 0/17/34 au + // lieu de 0/20/40, et le test mesurait alors la detection de + // scene plutot que `gop_size`. + for y in 0..H as usize { + let row = &mut planes[y * lay.bpr_y..y * lay.bpr_y + W as usize]; + for (x, px) in row.iter_mut().enumerate() { + *px = 100u8.wrapping_add(((x / 16 + y / 16 + i) % 40) as u8); + } + } + planes[lay.off_u..].fill(128); + VideoEncoder::copy_into(frame, &planes, W, H, W, H).expect("copy_into"); + (*frame).pts = i as i64; + averr(avcodec_send_frame(enc.ctx, frame), "send_frame").expect("send_frame"); + drain(pkt, &mut out); + } + enc.flush().expect("flush"); + drain(pkt, &mut out); + let mut pkt = pkt; + av_packet_free(&mut pkt); + let mut frame = frame; + av_frame_free(&mut frame); + } + Some(out) + } + + /// `profile_idc` porté par le SPS de l'extradata (Annex-B : `00 00 00 01 67 `). + fn profile_idc(extradata: &[u8]) -> Option { + extradata + .windows(6) + .find(|w| w[..4] == [0, 0, 0, 1] && w[4] & 0x1f == 7) + .map(|w| w[5]) + } + + /// L'export doit porter des images clés PÉRIODIQUES. + /// + /// Sans `(*ctx).gop_size`, le wrapper ffmpeg de `libopenh264` pose `g = -1` dans + /// ses `FFCodecDefault`, openh264 reçoit `uiIntraPeriod = 0` et n'émet qu'UNE + /// image clé pour tout le fichier — mesuré 1 image I sur 300 avant ce correctif. + /// Le MP4 reste lisible, mais tout seek redécode depuis le début et un paquet + /// abîmé emporte le reste. C'est le test qui aurait attrapé ça. + #[test] + fn l_export_h264_emet_des_images_cles_periodiques() { + let Some(enc) = encode(10, 45) else { + eprintln!("libopenh264 indisponible — test sauté"); + return; + }; + assert_eq!(enc.gop_size, 20, "gop_size doit valoir fps * 2"); + let keys: Vec = enc + .keyframes + .iter() + .enumerate() + .filter(|(_, k)| **k) + .map(|(i, _)| i) + .collect(); + assert_eq!( + keys, + vec![0, 20, 40], + "images clés attendues toutes les {} frames, obtenu {keys:?} sur {} paquets", + enc.gop_size, + enc.keyframes.len() + ); + } + + /// `tune_openh264` doit atteindre l'encodeur : fenêtre de QP explicite (sans quoi + /// openh264 remplace la sienne par (12, 42), cf. la doc de la fonction) et profil + /// High dans le SPS (sans quoi on expédie du Constrained Baseline en CAVLC). + #[test] + fn l_export_h264_configure_openh264() { + let Some(enc) = encode(10, 3) else { + eprintln!("libopenh264 indisponible — test sauté"); + return; + }; + assert_eq!((enc.qmin, enc.qmax), (1, 51), "fenêtre de QP non transmise"); + assert_eq!( + profile_idc(&enc.extradata), + Some(100), + "le SPS doit annoncer le profil High (100) ; 66 = Constrained Baseline, \ + le défaut du wrapper. extradata={:02x?}", + &enc.extradata[..enc.extradata.len().min(12)] + ); + } +} diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 10de3fac2..5721c2011 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -30,9 +30,10 @@ //! décodeurs, symétrique. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, + AacEncoder, PlanarPcm, }; +use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::compositor::Compositor; use crate::d3d::Gpu; use crate::timeline_walk::NextFrameTime; @@ -63,6 +64,20 @@ impl Drop for FrameGuard { /// seuil dépend du GOP des captures, pas du backend de décodage. const SEEK_FORWARD_MAX_SEC: f64 = 0.5; +/// Pourquoi ce décodeur est ouvert. La preview et l'export ne demandent pas la même chose +/// au décodeur, et sur macOS ils ne prennent donc pas le même backend. +/// +/// La preview lit au temps réel : il lui suffit de tenir la cadence, et elle scrube, donc la +/// latence d'un seek pèse plus que le débit. Une marche d'export déroule aussi vite que la +/// machine le permet — c'est du débit pur, et l'arbitrage n'est pas le même. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum DecodeIntent { + /// Lecture temps réel (`live.rs`). Arbitrage historique, inchangé. + Preview, + /// Marche d'export (`timeline_walk`, `gif_export`). + Export, +} + /// Décodeur ffmpeg — câblage VideoToolbox (et repli logiciel pour les codecs hors-session). /// Cf. `pipeline_windows::Decoder` pour la version D3D11VA. Mêmes champs publics pour /// que `live.rs::Player` reste portable ; les détails internes (hw_device_ctx, format @@ -97,7 +112,19 @@ pub struct Decoder { } impl Decoder { + /// Ouvre pour la PREVIEW. Signature conservée pour tous les appelants existants. pub fn open(path: &str, gpu: &Gpu) -> Result { + Self::open_with(path, gpu, DecodeIntent::Preview) + } + + /// Ouvre pour une marche d'EXPORT, où seul le débit compte. Windows et Linux exposent le + /// même point d'entrée sans rien en faire de particulier ; c'est ici qu'il change quelque + /// chose. + pub fn open_for_export(path: &str, gpu: &Gpu) -> Result { + Self::open_with(path, gpu, DecodeIntent::Export) + } + + pub fn open_with(path: &str, gpu: &Gpu, intent: DecodeIntent) -> Result { unsafe { let mut fmt: *mut crate::ffi::AVFormatContext = ptr::null_mut(); let cpath = CString::new(path)?; @@ -162,11 +189,58 @@ impl Decoder { const FF_PROFILE_H264_CONSTRAINED_BASELINE: i32 = 578; let is_baseline = profile == FF_PROFILE_H264_BASELINE || profile == FF_PROFILE_H264_CONSTRAINED_BASELINE; + // H.264 8 bits 4:2:0 : ce que produit toute capture d'écran, et le SEUL cas sur + // lequel l'arbitrage ci-dessous a été mesuré. `format` vient de `codecpar`, donc + // rempli par `avformat_find_stream_info` ; un flux dont le format reste inconnu + // n'est pas éligible et garde le comportement d'avant. + let is_h264_8bit = (*codecpar).codec_id == crate::ffi::AVCodecID::AV_CODEC_ID_H264 + && (*codecpar).format == crate::ffi::AVPixelFormat::AV_PIX_FMT_YUV420P as i32; let forced = std::env::var("OPENSCREEN_MAC_DECODE").ok(); let want_hw = match forced.as_deref() { Some("software") => false, Some("videotoolbox") => true, - _ => !is_baseline, + // Baseline : arbitrage historique, inchangé (cf. la note ci-dessus). + _ if is_baseline => false, + // MESURÉ, et contraire à ce que la note ci-dessus annonçait. Sur une marche + // d'export, un flux H.264 8 bits se décode plus vite en logiciel que par + // VideoToolbox — y compris en profil High, que cette note donnait à VT. + // + // Mac mini M1 8 Go / macOS 26.5. Source 1920x1080@60, 60 s, profil High. + // Scénario S4 du benchmark, sortie 1080p60 H.264. Trois cycles, un floor + // ffmpeg intercalé par cycle, dérive de fermeture 1,0002, machine à 86 % idle : + // + // VideoToolbox 32 079 ms 1,819x floor (MAD 34 ms) + // logiciel 22 863 ms 1,296x floor (MAD 16 ms) -28,7 % + // + // Par étage : décodage écran 13,13 s -> 1,02 s, webcam 4,20 s -> 0,29 s. + // L'image ne bouge pas — bitstream H.264 (NAL SEI retirés), pixels décodés et + // audio ont le même md5 sur les six sorties des deux variantes. + // + // La raison est celle que la note Baseline donne déjà, et elle ne dépend pas + // du profil : VideoToolbox a une latence FIXE par frame et alloue un + // CVPixelBuffer à chacune, là où le décodeur logiciel étale le travail sur des + // cœurs qui sont multiples. Ce qui compte est que la frame soit assez bon + // marché à décoder — ce que du 1080p 8 bits est. + // + // LA 4K AUSSI, mesurée depuis. Décodage seul, 1200 frames, meilleur de trois + // passes, même machine — avec le cas 1080p en témoin pour valider la méthode + // contre le résultat bout-en-bout ci-dessus : + // + // 1080p logiciel 2586 fps VideoToolbox 212 fps x12,2 + // 4K logiciel 849 fps VideoToolbox 71 fps x11,9 + // + // Le rapport ne bouge quasiment pas avec la résolution : la latence fixe par + // frame de VideoToolbox domine des deux côtés. Il n'y a donc pas de seuil de + // résolution à poser, et en poser un « par prudence » écarterait le chemin + // rapide du cas qui en profite le plus — 71 fps, c'est en dessous du temps + // réel pour une timeline 4K60. + // + // RESTE NON MESURÉ : 10 bits et HEVC. Ils gardent VideoToolbox, et la + // condition les écarte par construction (`format == YUV420P` et + // `codec_id == H264`). La preview aussi n'a pas été mesurée, et la changer + // sans la mesurer serait exactement l'erreur que ce commit corrige. + _ if intent == DecodeIntent::Export && is_h264_8bit => false, + _ => true, }; let r = if want_hw { crate::ffi::av_hwdevice_ctx_create( @@ -179,6 +253,18 @@ impl Decoder { } else { -1 // repli logiciel délibéré, pas un échec }; + // Dire lequel a été pris. Sans cette ligne, « l'export est lent » et « l'export a + // pris VideoToolbox » ne se distinguent pas dans un rapport de bug, et un + // changement d'arbitrage ne se vérifie qu'au chronomètre. + eprintln!( + "[pipeline] décodage {} : {} (codec={} profil={} format={} intention={:?})", + path.rsplit('/').next().unwrap_or(path), + if r == 0 { "videotoolbox" } else { "logiciel" }, + (*codecpar).codec_id, + profile, + (*codecpar).format, + intent, + ); let cpu = if r != 0 { // Pas de VideoToolbox sur ce codec : fallback software. `get_format` est // laissé à NULL (libavcodec choisit son format de sortie, ici NV12 via @@ -852,14 +938,18 @@ impl VideoEncoder { if self.sw.is_null() { // Chemin zero-copy : une frame du pool VideoToolbox, dont `data[3]` porte le // `CVPixelBuffer` dans lequel le compositeur va rendre directement. - let frame = crate::ffi::av_frame_alloc(); - if frame.is_null() { - bail!("av_frame_alloc (frame VT)"); - } - let mut frame = frame; - if crate::ffi::av_hwframe_get_buffer((*self.ctx).hw_frames_ctx, frame, 0) < 0 { - crate::ffi::av_frame_free(&mut frame); - bail!("av_hwframe_get_buffer (pool VT épuisé)"); + let mut frame; + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::VtGetBuffer); + let f = crate::ffi::av_frame_alloc(); + if f.is_null() { + bail!("av_frame_alloc (frame VT)"); + } + frame = f; + if crate::ffi::av_hwframe_get_buffer((*self.ctx).hw_frames_ctx, frame, 0) < 0 { + crate::ffi::av_frame_free(&mut frame); + bail!("av_hwframe_get_buffer (pool VT épuisé)"); + } } let pb = (*frame).data[3] as *mut std::ffi::c_void; if pb.is_null() { @@ -872,10 +962,13 @@ impl VideoEncoder { return Err(e); } (*frame).pts = pts; - let sent = crate::ffi::averr( - crate::ffi::avcodec_send_frame(self.ctx, frame), - "send_frame_composited_vt", - ); + let sent = { + let _p = crate::export_probe::scope(crate::export_probe::Stage::SendFrame); + crate::ffi::averr( + crate::ffi::avcodec_send_frame(self.ctx, frame), + "send_frame_composited_vt", + ) + }; crate::ffi::av_frame_free(&mut frame); return sent; } @@ -897,6 +990,7 @@ impl VideoEncoder { (*self.sw).linesize[1] as usize, )?; (*self.sw).pts = pts; + let _p = crate::export_probe::scope(crate::export_probe::Stage::SendFrame); crate::ffi::averr( crate::ffi::avcodec_send_frame(self.ctx, self.sw), "send_frame_composited", @@ -998,6 +1092,7 @@ pub fn run_composited_multi( bail!("run_composited_multi: aucun clip à exporter"); } let (out_w, out_h) = (params.width, params.height); + crate::export_probe::reset(); let t0 = std::time::Instant::now(); let mut frames: u64 = 0; @@ -1064,7 +1159,7 @@ pub fn run_composited_multi( } // Un PCM par clip, assemblé après la marche vidéo : c'est elle qui dit combien de // frames chaque clip a réellement produit, donc combien d'audio lui revient. - let mut clip_pcm: Vec> = (0..clips.len()).map(|_| None).collect(); + let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let mut clip_frame_counts: Vec = vec![0; clips.len()]; let mut opkt = unsafe { crate::ffi::av_packet_alloc() }; @@ -1077,6 +1172,11 @@ pub fn run_composited_multi( // raconte avoir déjà coûté une fois. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); frames = unsafe { crate::timeline_walk::walk_composited_timeline( clips, @@ -1089,29 +1189,38 @@ pub fn run_composited_multi( &mut webcam_decs, &mut |n| { enc.send_composited(comp, out_w, out_h, n as i64)?; - drain_encoder(ectx, octx, ostream, opkt)?; - progress(n + 1); + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::DrainMux); + drain_encoder(ectx, octx, ostream, opkt)?; + } + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::Progress); + progress(n + 1); + } Ok(()) }, &mut |clip_index, source_end_sec, frames_in_clip, speed_segments| { clip_frame_counts[clip_index] = frames_in_clip; let clip = &clips[clip_index]; if clip.has_audio && frames_in_clip > 0 { - match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { - Ok(Some(pcm)) => { - clip_pcm[clip_index] = Some(stretch_clip_pcm_by_speed( - &pcm, - speed_segments, - out_fps as f64, - )); - } - Ok(None) => eprintln!( - "[pipeline] warning: clip #{clip_index} déclaré audio mais sans flux décodable; silence conservé", - ), - Err(error) => eprintln!( - "[pipeline] warning: décodage audio du clip #{clip_index} échoué ({error:#}); silence conservé", - ), - } + // L'audio d'un clip ne dépend que de ce clip : le décoder et l'étirer ici, + // sur le thread de rendu, immobilisait la barre d'export pour toute sa + // durée — rien n'appelle `progress()` entre deux clips. Le travail part + // sur un thread et se recouvre avec la composition du clip suivant ; les + // résultats sont récupérés après le parcours, rangés par index de clip. + let path = clip.screen.clone(); + let source_start_sec = clip.source_start_sec; + let segments = speed_segments.to_vec(); + audio_jobs.spawn(clip_index, move || { + decode_and_stretch_clip_audio( + clip_index, + &path, + source_start_sec, + source_end_sec, + &segments, + out_fps as f64, + ) + }); } Ok(()) }, @@ -1119,6 +1228,7 @@ pub fn run_composited_multi( }; // Flush : un null frame à l'encodeur finalise son bitstream. + let _finalize = crate::export_probe::scope(crate::export_probe::Stage::Finalize); unsafe { crate::ffi::averr( crate::ffi::avcodec_send_frame(ectx, ptr::null_mut()), @@ -1129,10 +1239,23 @@ pub fn run_composited_multi( // Le plan part des frames RÉELLEMENT produites par clip, pas des durées demandées : // un clip raccourci (source plus courte que sa borne) doit voir son audio raccourci // d'autant, sinon la piste dérive pour tous les suivants. + // Récupération des jobs audio lancés pendant le parcours. `spawn` en admet quatre + // avant d'en collecter un, donc il en reste au plus quatre à attendre ici — bornés + // par le plus lent, pas par leur somme ; les autres se sont recouverts avec + // l'encodage vidéo. + let clip_pcm: Vec> = audio_jobs + .into_results() + .into_iter() + .map(|slot| slot.flatten()) + .collect(); + let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; @@ -1146,6 +1269,8 @@ pub fn run_composited_multi( } let wall_s = t0.elapsed().as_secs_f64(); + drop(_finalize); + crate::export_probe::report(wall_s, frames); Ok(Stats { frames, wall_s, diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index 11738bcd4..67bd325d9 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -3,9 +3,10 @@ //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, + AacEncoder, PlanarPcm, }; +use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::compositor::{Compositor, OUT_H, OUT_W}; use crate::config::Cfg; use crate::cpu_frames::CpuFrames; @@ -25,6 +26,154 @@ use std::ptr; use std::time::Instant; use windows::core::Interface; +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DecodeFrameTestFault { + AfterAllocations, + PacketAllocNull, + FrameAllocNull, + CloneNull, + EofSendError, + AttachBufferRefNull, +} + +#[cfg(test)] +thread_local! { + static DECODE_FRAME_TEST_FAULT: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + static DECODE_FRAME_TEST_PACKET_RELEASED: std::cell::RefCell< + Option> + > = const { std::cell::RefCell::new(None) }; + static DECODE_FRAME_TEST_FRAME_RELEASED: std::cell::RefCell< + Option> + > = const { std::cell::RefCell::new(None) }; + static DECODE_FRAME_TEST_HWDEV_OBSERVER: std::cell::Cell<*mut AVBufferRef> = + const { std::cell::Cell::new(ptr::null_mut()) }; +} + +#[cfg(test)] +unsafe extern "C" fn observe_test_buffer_release(opaque: *mut c_void, data: *mut u8) { + let released = Box::from_raw(opaque as *mut std::sync::Arc); + released.store(true, std::sync::atomic::Ordering::SeqCst); + drop(Box::from_raw(data)); +} + +#[cfg(test)] +unsafe fn install_decode_frame_lifetime_probes( + hwdev: *mut AVBufferRef, + pkt: *mut AVPacket, + frame: *mut AVFrame, +) -> Result<()> { + let should_fail = DECODE_FRAME_TEST_FAULT + .with(|fault| fault.get() == Some(DecodeFrameTestFault::AfterAllocations)); + if !should_fail { + return Ok(()); + } + + let packet_released = DECODE_FRAME_TEST_PACKET_RELEASED.with(|signal| { + signal + .borrow() + .as_ref() + .expect("packet release signal") + .clone() + }); + let frame_released = DECODE_FRAME_TEST_FRAME_RELEASED.with(|signal| { + signal + .borrow() + .as_ref() + .expect("frame release signal") + .clone() + }); + let packet_data = Box::into_raw(Box::new(0u8)); + let packet_opaque = Box::into_raw(Box::new(packet_released)); + let packet_buf = av_buffer_create( + packet_data, + 1, + Some(observe_test_buffer_release), + packet_opaque as *mut c_void, + 0, + ); + if packet_buf.is_null() { + drop(Box::from_raw(packet_data)); + drop(Box::from_raw(packet_opaque)); + bail!("test av_buffer_create(packet)"); + } + (*pkt).buf = packet_buf; + (*pkt).data = packet_data; + (*pkt).size = 1; + + let frame_data = Box::into_raw(Box::new(0u8)); + let frame_opaque = Box::into_raw(Box::new(frame_released)); + let frame_buf = av_buffer_create( + frame_data, + 1, + Some(observe_test_buffer_release), + frame_opaque as *mut c_void, + 0, + ); + if frame_buf.is_null() { + drop(Box::from_raw(frame_data)); + drop(Box::from_raw(frame_opaque)); + bail!("test av_buffer_create(frame)"); + } + (*frame).buf[0] = frame_buf; + (*frame).data[0] = frame_data; + + let observer = av_buffer_ref(hwdev); + if observer.is_null() { + bail!("test av_buffer_ref(hwdev)"); + } + DECODE_FRAME_TEST_HWDEV_OBSERVER.with(|slot| slot.set(observer)); + bail!("injected failure after decode allocations") +} + +#[cfg(test)] +fn decode_frame_test_fault_is(expected: DecodeFrameTestFault) -> bool { + DECODE_FRAME_TEST_FAULT.with(|fault| fault.get() == Some(expected)) +} + +unsafe fn decode_packet_alloc() -> *mut AVPacket { + #[cfg(test)] + if decode_frame_test_fault_is(DecodeFrameTestFault::PacketAllocNull) { + return ptr::null_mut(); + } + av_packet_alloc() +} + +unsafe fn decode_frame_alloc() -> *mut AVFrame { + #[cfg(test)] + if decode_frame_test_fault_is(DecodeFrameTestFault::FrameAllocNull) { + return ptr::null_mut(); + } + av_frame_alloc() +} + +unsafe fn clone_decoded_frame(frame: *const AVFrame) -> *mut AVFrame { + #[cfg(test)] + if decode_frame_test_fault_is(DecodeFrameTestFault::CloneNull) { + return ptr::null_mut(); + } + av_frame_clone(frame) +} + +unsafe fn send_decode_eof(dctx: *mut AVCodecContext) -> i32 { + #[cfg(test)] + if decode_frame_test_fault_is(DecodeFrameTestFault::EofSendError) { + return AVERROR_INVALIDDATA; + } + avcodec_send_packet(dctx, ptr::null()) +} + +unsafe fn ref_decode_hw_device(hwdev: *const AVBufferRef) -> *mut AVBufferRef { + #[cfg(test)] + if decode_frame_test_fault_is(DecodeFrameTestFault::AttachBufferRefNull) { + let observer = av_buffer_ref(hwdev); + DECODE_FRAME_TEST_HWDEV_OBSERVER.with(|slot| slot.set(observer)); + return ptr::null_mut(); + } + av_buffer_ref(hwdev) +} + // Macros libav non générées par bindgen (function-like). Valeurs Windows/MSVC. // `AVERROR(EAGAIN)` dépend de la plateforme (cf. `crate::ffi`) ; ce fichier est // Windows-only, mais garder une troisième copie de la valeur est ce qui a laissé @@ -57,6 +206,13 @@ impl Drop for FrameGuard { } } +struct PacketGuard(*mut AVPacket); +impl Drop for PacketGuard { + fn drop(&mut self) { + unsafe { av_packet_free(&mut self.0) }; + } +} + /// Décode la n-ième frame d'une source sur NOTRE device (textures échantillonnables). /// Sert le harnais de composition (S3+), hors mesure. Retourne une frame indépendante. pub fn decode_frame_n(path: &str, gpu: &Gpu, n: u32) -> Result { @@ -70,49 +226,77 @@ unsafe fn decode_frame_n_inner(path: &str, gpu: &Gpu, n: u32) -> Result Result bool { + codec_id == AVCodecID::AV_CODEC_ID_H264 +} + +unsafe fn require_decoder_id( + codec_id: AVCodecID::Type, +) -> Result<(*const AVCodec, *mut AVCodecContext)> { + let dec = avcodec_find_decoder(codec_id); + if dec.is_null() { + bail!("no decoder for codec_id {}", codec_id as i32); + } + let dctx = avcodec_alloc_context3(dec); + if dctx.is_null() { + bail!("avcodec_alloc_context3"); + } + Ok((dec, dctx)) +} + +unsafe fn require_decoder( + codecpar: *mut AVCodecParameters, +) -> Result<(*const AVCodec, *mut AVCodecContext)> { + if codecpar.is_null() { + bail!("codecpar null"); + } + require_decoder_id((*codecpar).codec_id) +} + +unsafe fn attach_d3d11va(dctx: *mut AVCodecContext, gpu: &Gpu) -> Result<*mut AVBufferRef> { + let mut hwdev = av_hwdevice_ctx_alloc(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA); + if hwdev.is_null() { + bail!("av_hwdevice_ctx_alloc"); + } + let hwdc = (*hwdev).data as *mut AVHWDeviceContext; + let d3dctx = (*hwdc).hwctx as *mut AVD3D11VADeviceContext; + let dev_clone = gpu.device.clone(); + (*d3dctx).device = dev_clone.as_raw() as *mut ID3D11Device; + std::mem::forget(dev_clone); + if let Err(error) = averr(av_hwdevice_ctx_init(hwdev), "hwdevice_ctx_init") { + av_buffer_unref(&mut hwdev); + return Err(error); + } + let dctx_hwdev = ref_decode_hw_device(hwdev); + if dctx_hwdev.is_null() { + av_buffer_unref(&mut hwdev); + bail!("av_buffer_ref(hw_device_ctx)"); + } + (*dctx).hw_device_ctx = dctx_hwdev; + (*dctx).get_format = Some(get_hw_format); + Ok(hwdev) +} + // D3D11_TEXTURE2D_DESC.BindFlags (valeurs SDK) const D3D11_BIND_SHADER_RESOURCE: u32 = 0x8; const D3D11_BIND_DECODER: u32 = 0x200; @@ -229,10 +465,18 @@ unsafe fn run_c0_inner(screen: &str, out: &str, gpu: &Gpu) -> Result { avformat_open_input(&mut fmt, cpath.as_ptr(), ptr::null_mut(), ptr::null_mut()), "avformat_open_input", )?; - averr(avformat_find_stream_info(fmt, ptr::null_mut()), "find_stream_info")?; + let mut resources = DecoderOpenResources { + fmt, + dctx: ptr::null_mut(), + hwdev: ptr::null_mut(), + }; + averr( + avformat_find_stream_info(resources.fmt, ptr::null_mut()), + "find_stream_info", + )?; let vidx = av_find_best_stream( - fmt, + resources.fmt, AVMediaType::AVMEDIA_TYPE_VIDEO, -1, -1, @@ -242,33 +486,30 @@ unsafe fn run_c0_inner(screen: &str, out: &str, gpu: &Gpu) -> Result { if vidx < 0 { bail!("aucun flux vidéo"); } - let stream = sn_fmt_stream(fmt, vidx); + let stream = sn_fmt_stream(resources.fmt, vidx); let codecpar = (*stream).codecpar; - // ---- décodeur D3D11VA sur NOTRE device ---- - let dec = avcodec_find_decoder((*codecpar).codec_id); - if dec.is_null() { - bail!("décodeur introuvable"); - } - let dctx = avcodec_alloc_context3(dec); - averr(avcodec_parameters_to_context(dctx, codecpar), "params_to_ctx")?; - allow_d3d11va_h264_baseline(dctx); - - let hwdev = av_hwdevice_ctx_alloc(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA); - if hwdev.is_null() { - bail!("av_hwdevice_ctx_alloc"); + // ---- décodeur D3D11VA sur NOTRE device (H.264 only; see d3d11va_for_codec) ---- + if !d3d11va_for_codec((*codecpar).codec_id) { + bail!( + "C0 D3D11VA only supports H.264 (codec_id {})", + (*codecpar).codec_id as i32 + ); } - let hwdc = (*hwdev).data as *mut AVHWDeviceContext; - let d3dctx = (*hwdc).hwctx as *mut AVD3D11VADeviceContext; - // AddRef : ffmpeg Release ce device au teardown. On garde un +1 en fuyant un clone. - let dev_clone = gpu.device.clone(); - (*d3dctx).device = dev_clone.as_raw() as *mut ID3D11Device; - std::mem::forget(dev_clone); - averr(av_hwdevice_ctx_init(hwdev), "hwdevice_ctx_init")?; + let (dec, dctx) = require_decoder(codecpar)?; + resources.dctx = dctx; + averr( + avcodec_parameters_to_context(resources.dctx, codecpar), + "params_to_ctx", + )?; + allow_d3d11va_h264_baseline(resources.dctx); - (*dctx).hw_device_ctx = av_buffer_ref(hwdev); - (*dctx).get_format = Some(get_hw_format); - averr(avcodec_open2(dctx, dec, ptr::null_mut()), "avcodec_open2(dec)")?; + resources.hwdev = attach_d3d11va(resources.dctx, gpu)?; + averr( + avcodec_open2(resources.dctx, dec, ptr::null_mut()), + "avcodec_open2(dec)", + )?; + let (mut fmt, dctx, hwdev) = resources.into_raw(); // ---- encodeur (ouvert paresseusement à la 1re frame : il lui faut ses dims + hw_frames_ctx) ---- let mut enc: Option = None; @@ -493,6 +734,37 @@ pub(crate) struct Decoder { has_peek: bool, } +/// Owns the FFmpeg resources allocated while `Decoder::open` is still fallible. +/// Once a complete `Decoder` exists, `into_raw` transfers the same three pointers +/// to it and disarms this guard so exactly one Drop path remains responsible. +struct DecoderOpenResources { + fmt: *mut AVFormatContext, + dctx: *mut AVCodecContext, + hwdev: *mut AVBufferRef, +} + +impl DecoderOpenResources { + unsafe fn cleanup(&mut self) { + avcodec_free_context(&mut self.dctx); + av_buffer_unref(&mut self.hwdev); + avformat_close_input(&mut self.fmt); + } + + unsafe fn into_raw(mut self) -> (*mut AVFormatContext, *mut AVCodecContext, *mut AVBufferRef) { + let resources = (self.fmt, self.dctx, self.hwdev); + self.fmt = ptr::null_mut(); + self.dctx = ptr::null_mut(); + self.hwdev = ptr::null_mut(); + resources + } +} + +impl Drop for DecoderOpenResources { + fn drop(&mut self) { + unsafe { self.cleanup() }; + } +} + // SAFETY: `Decoder` only owns FFI pointers into FFmpeg's own heap-allocated state, which // has no OS thread affinity — safe to create on one thread and hand off to another as long // as it's touched from a single thread at a time (never concurrently), which is exactly the @@ -501,6 +773,13 @@ pub(crate) struct Decoder { unsafe impl Send for Decoder {} impl Decoder { + /// Même point d'entrée que sur macOS, pour que `timeline_walk` reste portable. Ici le + /// choix D3D11VA/logiciel dépend du feature level du device, pas de l'usage : l'intention + /// n'a rien à trancher. + pub(crate) unsafe fn open_for_export(path: &str, gpu: &Gpu) -> Result { + Self::open(path, gpu) + } + pub(crate) unsafe fn open(path: &str, gpu: &Gpu) -> Result { let mut fmt: *mut AVFormatContext = ptr::null_mut(); let cpath = CString::new(path)?; @@ -508,47 +787,56 @@ impl Decoder { avformat_open_input(&mut fmt, cpath.as_ptr(), ptr::null_mut(), ptr::null_mut()), "open_input", )?; - averr(avformat_find_stream_info(fmt, ptr::null_mut()), "find_stream_info")?; - let vidx = av_find_best_stream(fmt, AVMediaType::AVMEDIA_TYPE_VIDEO, -1, -1, ptr::null_mut(), 0); + let mut resources = DecoderOpenResources { + fmt, + dctx: ptr::null_mut(), + hwdev: ptr::null_mut(), + }; + averr( + avformat_find_stream_info(resources.fmt, ptr::null_mut()), + "find_stream_info", + )?; + let vidx = av_find_best_stream( + resources.fmt, + AVMediaType::AVMEDIA_TYPE_VIDEO, + -1, + -1, + ptr::null_mut(), + 0, + ); if vidx < 0 { bail!("aucun flux vidéo dans {path}"); } - let stream = sn_fmt_stream(fmt, vidx); + let stream = sn_fmt_stream(resources.fmt, vidx); let codecpar = (*stream).codecpar; - let dec = avcodec_find_decoder((*codecpar).codec_id); - let dctx = avcodec_alloc_context3(dec); - averr(avcodec_parameters_to_context(dctx, codecpar), "params_to_ctx")?; - allow_d3d11va_h264_baseline(dctx); - - // Backend CPU : on n'attache AUCUN hw_device_ctx et on ne force pas `get_format`, - // donc libavcodec choisit son décodeur logiciel et sort en mémoire système. Passer - // le device WARP à D3D11VA ne marcherait pas de toute façon — WARP n'expose pas - // d'`ID3D11VideoDevice` (`tests/warp_device_cannot_decode.rs`). - let cpu = if gpu.backend == Backend::Cpu { - // `threads = 0` : libavcodec prend le nombre de cœurs. C'est le seul réglage - // qui compte vraiment ici — sans lui le décodage logiciel est mono-thread et - // le benchmark mesurerait surtout ça. - (*dctx).thread_count = 0; - Some(CpuFrames::new(gpu)?) - } else { - None - }; + let codec_id = (*codecpar).codec_id; + let (dec, dctx) = require_decoder(codecpar)?; + resources.dctx = dctx; + averr( + avcodec_parameters_to_context(resources.dctx, codecpar), + "params_to_ctx", + )?; + allow_d3d11va_h264_baseline(resources.dctx); - let hwdev = if cpu.is_some() { - ptr::null_mut() + // Hardware D3D11VA is the H.264 capture path. WARP has no video decoder + // (`tests/warp_device_cannot_decode.rs`). AV1/VP9 (legacy WebMs, #554) + // take the same software CpuFrames axis as Backend::Cpu. + let want_hw = gpu.backend != Backend::Cpu && d3d11va_for_codec(codec_id); + let cpu = if want_hw { + None } else { - let hwdev = av_hwdevice_ctx_alloc(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA); - let hwdc = (*hwdev).data as *mut AVHWDeviceContext; - let d3dctx = (*hwdc).hwctx as *mut AVD3D11VADeviceContext; - let dev_clone = gpu.device.clone(); - (*d3dctx).device = dev_clone.as_raw() as *mut ID3D11Device; - std::mem::forget(dev_clone); - averr(av_hwdevice_ctx_init(hwdev), "hwdevice_ctx_init")?; - (*dctx).hw_device_ctx = av_buffer_ref(hwdev); - (*dctx).get_format = Some(get_hw_format); - hwdev + (*resources.dctx).thread_count = 0; + Some(CpuFrames::new(gpu)?) }; - averr(avcodec_open2(dctx, dec, ptr::null_mut()), "avcodec_open2")?; + + if want_hw { + resources.hwdev = attach_d3d11va(resources.dctx, gpu)?; + } + averr( + avcodec_open2(resources.dctx, dec, ptr::null_mut()), + "avcodec_open2", + )?; + let (fmt, dctx, hwdev) = resources.into_raw(); Ok(Decoder { fmt, @@ -573,9 +861,18 @@ impl Decoder { /// `next`, donc les deux doivent rendre la même chose. Le temps (`cur_time_sec`), lui, /// continue de se lire sur la vraie frame décodée. pub(crate) fn cur_frame(&self) -> *mut AVFrame { - match &self.cpu { + let frame = match &self.cpu { Some(cpu) => cpu.current(), None => self.frame, + }; + // `AVFrame*` identifies the reusable container, not whether it currently contains a + // presentable frame. It stays allocated before the first decode and can be unreffed by + // a seek that runs to EOF. Returning that non-null shell let Player's webcam hold path + // feed a null D3D11 texture to the compositor on replay after #554's AV1 clip. + if frame.is_null() || unsafe { (*frame).data[0].is_null() } { + ptr::null_mut() + } else { + frame } } @@ -1343,6 +1640,11 @@ unsafe fn run_multi_inner( // fenêtrage par clip ; `walk_composited_timeline` s'en charge. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur @@ -1390,8 +1692,7 @@ unsafe fn run_multi_inner( let opkt = av_packet_alloc(); let mut clip_frame_counts = vec![0u64; clips.len()]; - let mut clip_pcm: Vec> = - std::iter::repeat_with(|| None).take(clips.len()).collect(); + let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let t0 = Instant::now(); let frames = walk_composited_timeline( @@ -1431,23 +1732,24 @@ unsafe fn run_multi_inner( clip_frame_counts[clip_index] = frames_in_clip; let clip = &clips[clip_index]; if clip.has_audio && frames_in_clip > 0 { - match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { - Ok(Some(pcm)) => { - clip_pcm[clip_index] = Some(stretch_clip_pcm_by_speed( - &pcm, - speed_segments, - out_fps as f64, - )); - } - Ok(None) => eprintln!( - "[pipeline] warning: clip #{} déclaré audio mais sans flux décodable; silence conservé", - clip_index, - ), - Err(error) => eprintln!( - "[pipeline] warning: décodage audio du clip #{} échoué ({error:#}); silence conservé", + // L'audio d'un clip ne dépend que de ce clip : le décoder et l'étirer ici, + // sur le thread de rendu, immobilisait la barre d'export pour toute sa durée + // — rien n'appelle `progress()` entre deux clips. Le travail part sur un + // thread et se recouvre avec la composition du clip suivant ; les résultats + // sont récupérés après le parcours, rangés par index de clip. + let path = clip.screen.clone(); + let source_start_sec = clip.source_start_sec; + let segments = speed_segments.to_vec(); + audio_jobs.spawn(clip_index, move || { + decode_and_stretch_clip_audio( clip_index, - ), - } + &path, + source_start_sec, + source_end_sec, + &segments, + out_fps as f64, + ) + }); } Ok(()) }, @@ -1460,6 +1762,15 @@ unsafe fn run_multi_inner( enc.send(ptr::null_mut())?; drain_encoder(ectx, octx, ostream, opkt)?; + // Récupération des jobs audio lancés pendant le parcours. `spawn` en admet quatre avant + // d'en collecter un, donc il en reste au plus quatre à attendre ici — bornés par le plus + // lent, pas par leur somme ; tous les autres se sont recouverts avec l'encodage. + let clip_pcm: Vec> = audio_jobs + .into_results() + .into_iter() + .map(|slot| slot.flatten()) + .collect(); + let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let audio_plan = build_audio_concat_plan( &clip_frame_counts, @@ -1467,7 +1778,7 @@ unsafe fn run_multi_inner( out_fps as f64, ); let assembled_audio = finish_audio( - assemble_concatenated_pcm(&clip_pcm, &audio_plan), + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks), audio_settings, ); audio_encoder.encode(&assembled_audio, octx)?; @@ -1496,6 +1807,141 @@ unsafe fn run_multi_inner( mod tests { use super::*; + struct DecodeFrameFaultReset; + + impl Drop for DecodeFrameFaultReset { + fn drop(&mut self) { + DECODE_FRAME_TEST_FAULT.with(|fault| fault.set(None)); + DECODE_FRAME_TEST_PACKET_RELEASED.with(|signal| *signal.borrow_mut() = None); + DECODE_FRAME_TEST_FRAME_RELEASED.with(|signal| *signal.borrow_mut() = None); + DECODE_FRAME_TEST_HWDEV_OBSERVER.with(|slot| unsafe { + let mut observer = slot.replace(ptr::null_mut()); + av_buffer_unref(&mut observer); + }); + } + } + + fn install_decode_frame_fault( + fault: DecodeFrameTestFault, + packet_released: std::sync::Arc, + frame_released: std::sync::Arc, + ) -> DecodeFrameFaultReset { + DECODE_FRAME_TEST_FAULT.with(|slot| slot.set(Some(fault))); + DECODE_FRAME_TEST_PACKET_RELEASED.with(|slot| *slot.borrow_mut() = Some(packet_released)); + DECODE_FRAME_TEST_FRAME_RELEASED.with(|slot| *slot.borrow_mut() = Some(frame_released)); + DecodeFrameFaultReset + } + + fn install_simple_decode_frame_fault(fault: DecodeFrameTestFault) -> DecodeFrameFaultReset { + install_decode_frame_fault( + fault, + std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + ) + } + + #[derive(Debug, PartialEq, Eq)] + enum HardwarePrerequisite { + Available, + Unsupported, + } + + const DXGI_ERROR_UNSUPPORTED_CODE: i32 = 0x887A_0004u32 as i32; + + fn classify_hardware_prerequisite( + result: windows::core::Result<()>, + ) -> windows::core::Result { + match result { + Ok(()) => Ok(HardwarePrerequisite::Available), + Err(error) if error.code().0 == DXGI_ERROR_UNSUPPORTED_CODE => { + Ok(HardwarePrerequisite::Unsupported) + } + Err(error) => Err(error), + } + } + + fn raw_hardware_prerequisite() -> windows::core::Result<()> { + use windows::Win32::Foundation::{E_UNEXPECTED, HMODULE}; + use windows::Win32::Graphics::Direct3D::{ + D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL, D3D_FEATURE_LEVEL_11_1, + }; + use windows::Win32::Graphics::Direct3D11::{ + D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, D3D11_CREATE_DEVICE_BGRA_SUPPORT, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, + }; + + let levels = [D3D_FEATURE_LEVEL_11_1]; + let mut device: Option = None; + let mut context: Option = None; + let mut got = D3D_FEATURE_LEVEL::default(); + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&levels), + D3D11_SDK_VERSION, + Some(&mut device), + Some(&mut got), + Some(&mut context), + )?; + } + if device.is_none() || context.is_none() || got != D3D_FEATURE_LEVEL_11_1 { + return Err(windows::core::Error::from(E_UNEXPECTED)); + } + Ok(()) + } + + fn strict_hardware_gpu(test_name: &str) -> Option { + match classify_hardware_prerequisite(raw_hardware_prerequisite()) { + Ok(HardwarePrerequisite::Unsupported) => { + println!( + "NOT_EXECUTED:{test_name}:raw D3D11 preflight returned DXGI_ERROR_UNSUPPORTED (0x887A0004)" + ); + None + } + Ok(HardwarePrerequisite::Available) => Some( + Gpu::create(false) + .unwrap_or_else(|error| panic!("{test_name}: strict Gpu::create failed after successful raw hardware preflight: {error:#}")), + ), + Err(error) => panic!( + "{test_name}: raw D3D11 hardware preflight failed with non-skippable HRESULT {:#010X}: {error}", + error.code().0 as u32 + ), + } + } + + fn decode_frame_error(path: &std::path::Path, gpu: &Gpu, n: u32) -> anyhow::Error { + match decode_frame_n(path.to_str().expect("utf8 path"), gpu, n) { + Ok(_) => panic!("decode_frame_n unexpectedly succeeded"), + Err(error) => error, + } + } + + #[test] + fn hardware_prerequisite_classification() { + use windows::Win32::Foundation::{E_INVALIDARG, E_OUTOFMEMORY, E_UNEXPECTED}; + + assert_eq!( + classify_hardware_prerequisite(Ok(())).expect("success classification"), + HardwarePrerequisite::Available + ); + let unsupported = + windows::core::Error::from(windows::core::HRESULT(DXGI_ERROR_UNSUPPORTED_CODE)); + assert_eq!( + classify_hardware_prerequisite(Err(unsupported)) + .expect("DXGI_ERROR_UNSUPPORTED classification"), + HardwarePrerequisite::Unsupported + ); + for hresult in [E_INVALIDARG, E_OUTOFMEMORY, E_UNEXPECTED] { + let error = windows::core::Error::from(hresult); + let returned = classify_hardware_prerequisite(Err(error)) + .expect_err("ordinary failures must remain failures"); + assert_eq!(returned.code(), hresult); + } + } + /// L'ordre EST le contrat : tous les candidats zéro-copie d'abord, ceux qui exigent la /// mémoire système ensuite (`*_qsv` et `*_mf` sont matériels eux aussi — ce qui les /// distingue est le format d'entrée, pas le silicium). Un candidat système remonté @@ -1589,6 +2035,831 @@ mod tests { } } } + + #[test] + fn d3d11va_is_h264_only() { + assert!(d3d11va_for_codec(AVCodecID::AV_CODEC_ID_H264)); + assert!(!d3d11va_for_codec(AVCodecID::AV_CODEC_ID_AV1)); + assert!(!d3d11va_for_codec(AVCodecID::AV_CODEC_ID_VP9)); + } + + #[test] + fn require_decoder_rejects_none() { + let err = unsafe { require_decoder_id(AVCodecID::AV_CODEC_ID_NONE) } + .expect_err("NONE must not allocate a context"); + let msg = format!("{err:#}"); + assert!(msg.contains("codec_id"), "{msg}"); + } + + fn select_ffmpeg_exe( + crate_dir: &std::path::Path, + configured_dir: Option, + ) -> Option { + let mut candidates = Vec::new(); + if let Some(dir) = configured_dir { + candidates.push(dir.join("bin").join("ffmpeg.exe")); + } + candidates + .push(crate_dir.join("../thirdparty/ffmpeg-n8.1.2-win64-lgpl-shared/bin/ffmpeg.exe")); + candidates.into_iter().find(|p| p.is_file()) + } + + fn ffmpeg_exe() -> std::path::PathBuf { + let crate_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + select_ffmpeg_exe( + &crate_dir, + std::env::var_os("FFMPEG_DIR").map(std::path::PathBuf::from), + ) + .unwrap_or_else(|| panic!("ffmpeg.exe not found next to FFMPEG_DIR / crates/thirdparty")) + } + + fn ffprobe_exe() -> std::path::PathBuf { + let ffprobe = ffmpeg_exe().with_file_name("ffprobe.exe"); + assert!( + ffprobe.is_file(), + "ffprobe.exe not found next to ffmpeg.exe: {ffprobe:?}" + ); + ffprobe + } + + fn encode_color(codec_args: &[&str], filename: &str) -> std::path::PathBuf { + encode_color_for_duration(codec_args, filename, "0.4") + } + + fn encode_color_for_duration( + codec_args: &[&str], + filename: &str, + duration_sec: &str, + ) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("openscreen-554-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let out = dir.join(filename); + let ff = ffmpeg_exe(); + let mut cmd = std::process::Command::new(&ff); + let input = format!("color=c=red:s=64x64:d={duration_sec}"); + cmd.args(["-y", "-f", "lavfi", "-i", input.as_str()]); + cmd.args(codec_args); + cmd.arg(&out); + let output = cmd.output().unwrap_or_else(|e| panic!("spawn {ff:?}: {e}")); + assert!( + output.status.success() && out.is_file(), + "ffmpeg {cmd:?} failed status={} stderr={}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + out + } + + unsafe fn allocated_decoder_open_resources(filename: &str) -> DecoderOpenResources { + let path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], filename); + let cpath = CString::new(path.to_string_lossy().as_bytes()).expect("fixture path CString"); + let mut fmt = ptr::null_mut(); + averr( + avformat_open_input(&mut fmt, cpath.as_ptr(), ptr::null_mut(), ptr::null_mut()), + "test open_input", + ) + .expect("open fixture"); + averr( + avformat_find_stream_info(fmt, ptr::null_mut()), + "test find_stream_info", + ) + .expect("find fixture streams"); + let vidx = av_find_best_stream( + fmt, + AVMediaType::AVMEDIA_TYPE_VIDEO, + -1, + -1, + ptr::null_mut(), + 0, + ); + assert!(vidx >= 0, "fixture video stream"); + let codecpar = (*sn_fmt_stream(fmt, vidx)).codecpar; + let (_, dctx) = require_decoder(codecpar).expect("fixture decoder context"); + let hwdev = av_hwdevice_ctx_alloc(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA); + assert!(!hwdev.is_null(), "test hardware-device context"); + DecoderOpenResources { fmt, dctx, hwdev } + } + + #[test] + fn decoder_open_resources_cleanup_nulls_every_owned_pointer() { + unsafe { + let mut resources = + allocated_decoder_open_resources("decoder-open-resources-cleanup.mp4"); + resources.cleanup(); + assert!(resources.dctx.is_null()); + assert!(resources.hwdev.is_null()); + assert!(resources.fmt.is_null()); + } + } + + #[test] + fn decoder_open_resources_release_transfers_every_pointer_once() { + unsafe { + let resources = allocated_decoder_open_resources("decoder-open-resources-release.mp4"); + let (mut fmt, mut dctx, mut hwdev) = resources.into_raw(); + assert!(!dctx.is_null()); + assert!(!hwdev.is_null()); + assert!(!fmt.is_null()); + avcodec_free_context(&mut dctx); + av_buffer_unref(&mut hwdev); + avformat_close_input(&mut fmt); + } + } + + #[test] + fn decode_frame_n_failure_paths_release_resources() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + let Some(gpu) = strict_hardware_gpu("decode_frame_n_failure_paths_release_resources") + else { + return; + }; + let path = encode_color( + &["-c:v", "libopenh264", "-b:v", "200k"], + "decode-frame-n-lifetime.mp4", + ); + let packet_released = Arc::new(AtomicBool::new(false)); + let frame_released = Arc::new(AtomicBool::new(false)); + let _fault = install_decode_frame_fault( + DecodeFrameTestFault::AfterAllocations, + Arc::clone(&packet_released), + Arc::clone(&frame_released), + ); + + let error = decode_frame_error(&path, &gpu, 0); + assert!( + format!("{error:#}").contains("injected failure after decode allocations"), + "unexpected injected error: {error:#}" + ); + let mut observer = + DECODE_FRAME_TEST_HWDEV_OBSERVER.with(|slot| slot.replace(ptr::null_mut())); + assert!( + !observer.is_null(), + "hardware-device observer was not installed" + ); + let hwdev_ref_count = unsafe { av_buffer_get_ref_count(observer) }; + let input_handle_released = std::fs::remove_file(&path).is_ok(); + let packet_was_released = packet_released.load(Ordering::SeqCst); + let frame_was_released = frame_released.load(Ordering::SeqCst); + unsafe { av_buffer_unref(&mut observer) }; + + println!( + "RELEASE_OBSERVATION:fmt_handle={input_handle_released}:hwdev_refs={hwdev_ref_count}:packet_callback={packet_was_released}:frame_callback={frame_was_released}" + ); + assert!( + input_handle_released + && hwdev_ref_count == 1 + && packet_was_released + && frame_was_released, + "UNRELEASED_RESOURCE fmt_handle={input_handle_released} hwdev_refs={hwdev_ref_count} packet_callback={packet_was_released} frame_callback={frame_was_released}" + ); + + let unsupported_path = encode_color( + &["-c:v", "libaom-av1", "-cpu-used", "8"], + "decode-frame-n-unsupported.webm", + ); + let unsupported_error = decode_frame_error(&unsupported_path, &gpu, 0); + let unsupported_message = format!("{unsupported_error:#}"); + assert!( + unsupported_message + .contains(&format!("codec_id {}", AVCodecID::AV_CODEC_ID_AV1 as i32)), + "unsupported codec id was not preserved before format teardown: {unsupported_message}" + ); + std::fs::remove_file(&unsupported_path).expect("unsupported input handle released"); + + for (fault, filename, expected) in [ + ( + DecodeFrameTestFault::PacketAllocNull, + "decode-frame-n-packet-null.mp4", + "av_packet_alloc", + ), + ( + DecodeFrameTestFault::FrameAllocNull, + "decode-frame-n-frame-null.mp4", + "av_frame_alloc", + ), + ( + DecodeFrameTestFault::CloneNull, + "decode-frame-n-clone-null.mp4", + "av_frame_clone", + ), + ( + DecodeFrameTestFault::EofSendError, + "decode-frame-n-eof-send.mp4", + "send_eof", + ), + ] { + let path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], filename); + let frame_number = if fault == DecodeFrameTestFault::EofSendError { + u32::MAX + } else { + 0 + }; + let error = { + let _fault = install_simple_decode_frame_fault(fault); + decode_frame_error(&path, &gpu, frame_number) + }; + let message = format!("{error:#}"); + assert!(message.contains(expected), "{fault:?}: {message}"); + std::fs::remove_file(&path) + .unwrap_or_else(|error| panic!("{fault:?}: input handle leaked: {error}")); + } + + let attach_path = encode_color( + &["-c:v", "libopenh264", "-b:v", "200k"], + "decode-frame-n-attach-ref-null.mp4", + ); + let attach_fault = + install_simple_decode_frame_fault(DecodeFrameTestFault::AttachBufferRefNull); + let attach_error = decode_frame_error(&attach_path, &gpu, 0); + assert!( + format!("{attach_error:#}").contains("av_buffer_ref(hw_device_ctx)"), + "unexpected attach error: {attach_error:#}" + ); + let mut attach_observer = + DECODE_FRAME_TEST_HWDEV_OBSERVER.with(|slot| slot.replace(ptr::null_mut())); + assert!( + !attach_observer.is_null(), + "attach observer was not installed" + ); + let attach_refs = unsafe { av_buffer_get_ref_count(attach_observer) }; + unsafe { av_buffer_unref(&mut attach_observer) }; + drop(attach_fault); + std::fs::remove_file(&attach_path).expect("attach-ref-null input handle released"); + assert_eq!( + attach_refs, 1, + "UNRELEASED_RESOURCE attach_d3d11va local hwdev refs={attach_refs}" + ); + println!("FAILURE_PATH_ASSERTIONS_COMPLETED"); + } + + #[test] + fn decode_frame_n_returned_frame_keeps_its_buffers() { + let Some(gpu) = strict_hardware_gpu("decode_frame_n_returned_frame_keeps_its_buffers") + else { + return; + }; + let path = encode_color( + &["-c:v", "libopenh264", "-b:v", "200k"], + "decode-frame-n-returned-frame.mp4", + ); + let frame = decode_frame_n(path.to_str().expect("utf8 path"), &gpu, 0) + .unwrap_or_else(|error| panic!("decode first H.264 frame: {error:#}")); + assert!(!frame.0.is_null(), "returned frame pointer"); + let source_buffer = unsafe { (*frame.0).buf[0] }; + assert!(!source_buffer.is_null(), "returned frame buffer reference"); + let mut observer = unsafe { av_buffer_ref(source_buffer) }; + assert!( + !observer.is_null(), + "observer reference for returned frame buffer" + ); + let refs_with_frame = unsafe { av_buffer_get_ref_count(observer) }; + drop(frame); + let refs_after_frame_drop = unsafe { av_buffer_get_ref_count(observer) }; + println!( + "RETURNED_FRAME_REFS:with_frame={refs_with_frame}:after_frame_drop={refs_after_frame_drop}" + ); + assert_eq!( + refs_after_frame_drop + 1, + refs_with_frame, + "FrameGuard must own one independent AVBuffer reference" + ); + assert!( + refs_after_frame_drop >= 1, + "observer reference must remain valid" + ); + unsafe { av_buffer_unref(&mut observer) }; + std::fs::remove_file(path).expect("returned-frame input handle released"); + } + + #[test] + fn ffmpeg_dir_selects_the_configured_executable_over_the_builtin() { + let crate_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let built_in = + crate_dir.join("../thirdparty/ffmpeg-n8.1.2-win64-lgpl-shared/bin/ffmpeg.exe"); + assert!( + built_in.is_file(), + "built-in control is missing: {built_in:?}" + ); + + let configured_dir = std::env::temp_dir().join(format!( + "openscreen-554-ffmpeg-override-{}", + std::process::id() + )); + let configured = configured_dir.join("bin").join("ffmpeg.exe"); + std::fs::create_dir_all(configured.parent().expect("configured ffmpeg parent")) + .expect("create configured ffmpeg directory"); + std::fs::File::create(&configured).expect("create configured ffmpeg executable"); + + let selected = select_ffmpeg_exe(&crate_dir, Some(configured_dir.clone())) + .expect("select configured ffmpeg executable"); + assert_eq!( + selected, configured, + "an explicit FFMPEG_DIR must override the built-in test fixture executable" + ); + std::fs::remove_dir_all(configured_dir).expect("remove configured ffmpeg directory"); + } + + #[derive(Clone, Copy, Debug)] + struct EbmlElement { + id: u64, + start: usize, + size_offset: usize, + size_width: usize, + data_start: usize, + data_end: usize, + unknown_size: bool, + } + + fn vint_width(first: u8, at: usize) -> usize { + let width = first.leading_zeros() as usize + 1; + assert!(width <= 8, "invalid EBML vint at offset {at:#x}"); + width + } + + fn read_ebml_id(bytes: &[u8], at: usize) -> (u64, usize) { + let first = *bytes + .get(at) + .unwrap_or_else(|| panic!("missing EBML id at {at:#x}")); + let width = vint_width(first, at); + assert!(width <= 4, "EBML id is wider than four bytes at {at:#x}"); + let end = at.checked_add(width).expect("EBML id offset overflow"); + let encoded = bytes + .get(at..end) + .unwrap_or_else(|| panic!("truncated EBML id at {at:#x}")); + let id = encoded + .iter() + .fold(0u64, |value, byte| (value << 8) | u64::from(*byte)); + (id, width) + } + + fn read_ebml_size(bytes: &[u8], at: usize) -> (Option, usize) { + let first = *bytes + .get(at) + .unwrap_or_else(|| panic!("missing EBML size at {at:#x}")); + let width = vint_width(first, at); + let end = at.checked_add(width).expect("EBML size offset overflow"); + let encoded = bytes + .get(at..end) + .unwrap_or_else(|| panic!("truncated EBML size at {at:#x}")); + let value_mask = if width == 8 { 0 } else { 0xffu8 >> width }; + let value = encoded[1..] + .iter() + .fold(u64::from(first & value_mask), |value, byte| { + (value << 8) | u64::from(*byte) + }); + let unknown_value = (1u64 << (7 * width)) - 1; + if value == unknown_value { + (None, width) + } else { + let value = usize::try_from(value).expect("EBML size does not fit usize"); + (Some(value), width) + } + } + + fn ebml_element_at(bytes: &[u8], start: usize, parent_end: usize) -> EbmlElement { + let (id, id_width) = read_ebml_id(bytes, start); + let size_offset = start + .checked_add(id_width) + .expect("EBML size offset overflow"); + let (size, size_width) = read_ebml_size(bytes, size_offset); + let data_start = size_offset + .checked_add(size_width) + .expect("EBML payload offset overflow"); + assert!( + data_start <= parent_end, + "EBML header exceeds parent at {start:#x}" + ); + let data_end = match size { + Some(size) => data_start + .checked_add(size) + .expect("EBML payload offset overflow"), + None => parent_end, + }; + assert!( + data_end <= parent_end, + "EBML element {id:#x} exceeds its parent" + ); + EbmlElement { + id, + start, + size_offset, + size_width, + data_start, + data_end, + unknown_size: size.is_none(), + } + } + + fn ebml_children(bytes: &[u8], start: usize, end: usize) -> Vec { + let mut children = Vec::new(); + let mut at = start; + while at < end { + let child = ebml_element_at(bytes, at, end); + assert!( + child.data_end > at, + "empty EBML element cannot advance at {at:#x}" + ); + children.push(child); + at = child.data_end; + if child.unknown_size { + assert_eq!( + at, end, + "unknown-sized child must consume the parent remainder" + ); + } + } + assert_eq!(at, end, "EBML children did not exactly fill their parent"); + children + } + + fn exactly_one(children: &[EbmlElement], id: u64, label: &str) -> EbmlElement { + let found: Vec<_> = children + .iter() + .copied() + .filter(|child| child.id == id) + .collect(); + assert_eq!( + found.len(), + 1, + "expected exactly one {label}, found {}", + found.len() + ); + found[0] + } + + /// Turn the pinned ffmpeg's valid AV1 WebM into the three malformed-but-decodable + /// characteristics from #554. This is deliberately a structural EBML edit: a raw byte + /// search could hit an AV1 payload byte and produce a fixture that only looked relevant. + fn make_legacy_av1_fixture(filename: &str) -> std::path::PathBuf { + const SEGMENT_ID: u64 = 0x1853_8067; + const TRACKS_ID: u64 = 0x1654_ae6b; + const TRACK_ENTRY_ID: u64 = 0xae; + const CODEC_ID_ID: u64 = 0x86; + const DEFAULT_DURATION_ID: u64 = 0x23e383; + const CODEC_PRIVATE_ID: u64 = 0x63a2; + const CLUSTER_ID: u64 = 0x1f43_b675; + + // One frame is intentional: after DefaultDuration is removed, there is no second + // timestamp from which avformat_find_stream_info can infer a replacement frame rate. + let path = encode_color_for_duration( + &[ + "-c:v", + "libaom-av1", + "-cpu-used", + "8", + "-usage", + "realtime", + "-b:v", + "50k", + "-output_ts_offset", + "0.4", + ], + filename, + "0.04", + ); + let mut bytes = std::fs::read(&path).expect("read generated AV1 WebM"); + let original = bytes.clone(); + let top_level = ebml_children(&bytes, 0, bytes.len()); + let segment = exactly_one(&top_level, SEGMENT_ID, "Segment"); + assert!( + !segment.unknown_size, + "generated Segment must have a finite size" + ); + let segment_children = ebml_children(&bytes, segment.data_start, segment.data_end); + let tracks = exactly_one(&segment_children, TRACKS_ID, "Tracks"); + let cluster = exactly_one(&segment_children, CLUSTER_ID, "Cluster"); + assert!( + !cluster.unknown_size, + "generated Cluster must start with a finite size" + ); + + let track_entries: Vec<_> = ebml_children(&bytes, tracks.data_start, tracks.data_end) + .into_iter() + .filter(|child| child.id == TRACK_ENTRY_ID) + .collect(); + let av1_tracks: Vec<_> = track_entries + .iter() + .copied() + .filter(|entry| { + let children = ebml_children(&bytes, entry.data_start, entry.data_end); + children.iter().any(|child| { + child.id == CODEC_ID_ID && &bytes[child.data_start..child.data_end] == b"V_AV1" + }) + }) + .collect(); + assert_eq!(av1_tracks.len(), 1, "expected exactly one V_AV1 TrackEntry"); + let track_children = + ebml_children(&bytes, av1_tracks[0].data_start, av1_tracks[0].data_end); + let codec_private = exactly_one(&track_children, CODEC_PRIVATE_ID, "AV1 CodecPrivate"); + let default_duration = exactly_one(&track_children, DEFAULT_DURATION_ID, "DefaultDuration"); + + assert!( + codec_private.data_start < codec_private.data_end, + "empty AV1 CodecPrivate" + ); + assert_eq!( + bytes[codec_private.data_start], 0x81, + "pinned encoder's AV1CodecConfigurationRecord layout changed" + ); + bytes[codec_private.data_start] = 0xff; + + let default_duration_len = default_duration.data_end - default_duration.start; + assert!( + (3..=128).contains(&default_duration_len), + "DefaultDuration cannot be replaced by a one-byte-size Void" + ); + let void_payload_len = default_duration_len - 2; + assert!( + void_payload_len <= 126, + "one-byte Void size would become unknown" + ); + bytes[default_duration.start] = 0xec; + bytes[default_duration.start + 1] = 0x80 | void_payload_len as u8; + bytes[default_duration.start + 2..default_duration.data_end].fill(0); + + assert!( + (1..=8).contains(&cluster.size_width), + "invalid Cluster size width {}", + cluster.size_width + ); + bytes[cluster.size_offset] = 0xff >> (cluster.size_width - 1); + bytes[cluster.size_offset + 1..cluster.data_start].fill(0xff); + + assert_eq!( + bytes.len(), + original.len(), + "fixture patch must preserve file length" + ); + let allowed = [ + codec_private.data_start..codec_private.data_start + 1, + default_duration.start..default_duration.data_end, + cluster.size_offset..cluster.data_start, + ]; + let changed: Vec<_> = original + .iter() + .zip(&bytes) + .enumerate() + .filter_map(|(index, (before, after))| (before != after).then_some(index)) + .collect(); + assert!(!changed.is_empty(), "fixture patch changed no bytes"); + assert!( + changed + .iter() + .all(|index| allowed.iter().any(|range| range.contains(index))), + "fixture patch changed bytes outside the three intended EBML fields: {changed:?}" + ); + for range in &allowed { + assert!( + changed.iter().any(|index| range.contains(index)), + "fixture patch did not change intended range {range:?}" + ); + } + std::fs::write(&path, &bytes).expect("write structurally patched AV1 WebM"); + + let ffprobe = ffprobe_exe(); + let output = std::process::Command::new(&ffprobe) + .args([ + "-v", + "warning", + "-select_streams", + "v:0", + "-show_entries", + "stream=codec_name,avg_frame_rate", + "-of", + "default=noprint_wrappers=1", + ]) + .arg(&path) + .output() + .unwrap_or_else(|e| panic!("spawn {ffprobe:?}: {e}")); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "ffprobe failed status={} stdout={stdout} stderr={stderr}", + output.status + ); + assert!( + stdout.contains("codec_name=av1"), + "ffprobe stdout: {stdout}" + ); + assert!( + stdout.contains("avg_frame_rate=0/0"), + "ffprobe stdout: {stdout}" + ); + assert!( + stderr.contains("Unknown version 127 of AV1CodecConfigurationRecord"), + "ffprobe stderr: {stderr}" + ); + assert!( + stderr + .to_ascii_lowercase() + .contains("unknown-sized element"), + "ffprobe stderr: {stderr}" + ); + + if let Some(out) = std::env::var_os("OPENSCREEN_554_FIXTURE_OUT") { + let out = std::path::PathBuf::from(out); + if let Some(parent) = out.parent() { + std::fs::create_dir_all(parent).expect("create fixture output directory"); + } + std::fs::copy(&path, &out).expect("copy verified issue 554 fixture"); + println!("ISSUE554_FIXTURE={}", out.display()); + } + path + } + + unsafe fn first_decoded_frame(dec: &mut Decoder) -> *mut AVFrame { + let frame = dec + .next() + .unwrap_or_else(|e| panic!("Decoder::next: {e:#}")); + assert!(!frame.is_null(), "expected a decoded frame, got null (EOF)"); + assert!( + (*frame).width > 0 && (*frame).height > 0, + "decoded frame has no pixels ({}x{})", + (*frame).width, + (*frame).height + ); + frame + } + + #[test] + fn av1_webm_opens_on_software_path() { + let Some(gpu) = strict_hardware_gpu("av1_webm_opens_on_software_path") else { + return; + }; + assert_eq!(gpu.backend, Backend::Hardware); + let path = make_legacy_av1_fixture("tiny-legacy.webm"); + let mut dec = unsafe { Decoder::open(path.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("AV1 Decoder::open: {e:#}")); + assert!( + dec.cpu.is_some(), + "AV1 on Hardware must use CpuFrames, not D3D11VA" + ); + let target_sec = 0.4; + let frame = unsafe { dec.seek_to(target_sec) } + .unwrap_or_else(|e| panic!("legacy AV1 nonzero seek: {e:#}")); + assert!(!frame.is_null(), "legacy AV1 nonzero seek reached EOF"); + assert_eq!(unsafe { (*frame).width }, 64); + assert_eq!(unsafe { (*frame).height }, 64); + let pts = unsafe { (*frame).best_effort_timestamp }; + assert_ne!(pts, i64::MIN, "legacy AV1 presentation timestamp"); + let time_base = unsafe { dec.tb_sec() }; + let observed_sec = pts as f64 * time_base; + assert!( + observed_sec >= target_sec - time_base * 0.5, + "legacy AV1 seek landed before its nonzero source target: target={target_sec:.3}s observed={observed_sec:.3}s" + ); + println!("CORE_ASSERTIONS_COMPLETED:av1_webm_opens_on_software_path"); + } + + #[test] + fn av1_software_seek_preserves_the_nonzero_target_timestamp() { + let Some(gpu) = + strict_hardware_gpu("av1_software_seek_preserves_the_nonzero_target_timestamp") + else { + return; + }; + assert_eq!(gpu.backend, Backend::Hardware); + let path = encode_color_for_duration( + &[ + "-c:v", + "libaom-av1", + "-cpu-used", + "8", + "-usage", + "realtime", + "-g", + "100", + ], + "nonzero-seek-av1.webm", + "1.2", + ); + let mut dec = unsafe { Decoder::open(path.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("AV1 Decoder::open: {e:#}")); + assert!( + dec.cpu.is_some(), + "AV1 must use the software presentation path" + ); + + let target_sec = 0.72; + let frame = unsafe { dec.seek_to(target_sec) }.expect("seek to nonzero AV1 source time"); + assert!(!frame.is_null(), "nonzero AV1 seek reached EOF"); + let pts = unsafe { (*frame).best_effort_timestamp }; + assert_ne!( + pts, + i64::MIN, + "software presentation frame must retain the decoded timestamp" + ); + let time_base = unsafe { dec.tb_sec() }; + let observed_sec = pts as f64 * time_base; + assert!( + observed_sec >= target_sec - time_base * 0.5, + "seek accepted a pre-target frame: target={target_sec:.3}s observed={observed_sec:.3}s" + ); + println!( + "CORE_ASSERTIONS_COMPLETED:av1_software_seek_preserves_the_nonzero_target_timestamp" + ); + } + + #[test] + fn h264_opens_on_d3d11va() { + let Some(gpu) = strict_hardware_gpu("h264_opens_on_d3d11va") else { + return; + }; + assert_eq!(gpu.backend, Backend::Hardware); + let path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], "tiny.mp4"); + let mut dec = unsafe { Decoder::open(path.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("H.264 Decoder::open: {e:#}")); + assert!( + dec.cpu.is_none(), + "H.264 on Hardware must keep D3D11VA" + ); + unsafe { first_decoded_frame(&mut dec) }; + println!("CORE_ASSERTIONS_COMPLETED:h264_opens_on_d3d11va"); + } + + #[test] + fn current_frame_requires_pixels_and_recovers_after_eof_seek() { + let Some(gpu) = + strict_hardware_gpu("current_frame_requires_pixels_and_recovers_after_eof_seek") + else { + return; + }; + assert_eq!(gpu.backend, Backend::Hardware); + let path = encode_color( + &["-c:v", "libopenh264", "-b:v", "200k"], + "current-frame.mp4", + ); + let mut dec = unsafe { Decoder::open(path.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("H.264 Decoder::open: {e:#}")); + + assert!( + dec.cur_frame().is_null(), + "allocated AVFrame without pixels is not current" + ); + unsafe { first_decoded_frame(&mut dec) }; + assert!( + !dec.cur_frame().is_null(), + "decoded frame must be presentable" + ); + + let unavailable = unsafe { dec.seek_to(10.0) }.expect("seek beyond EOF must not error"); + assert!( + unavailable.is_null(), + "seek beyond EOF must report no target frame" + ); + assert!( + dec.cur_frame().is_null(), + "unreffed AVFrame shell after EOF must not be exposed to the compositor" + ); + + let recovered = unsafe { dec.seek_to(0.0) }.expect("seek back to start"); + assert!( + !recovered.is_null(), + "seek back to start must decode a frame" + ); + assert!( + !dec.cur_frame().is_null(), + "recovered frame must be presentable" + ); + println!( + "CORE_ASSERTIONS_COMPLETED:current_frame_requires_pixels_and_recovers_after_eof_seek" + ); + } + + /// Playhead crossing clips is `Decoder::open` of the next source on the + /// same `Gpu` (#554). H.264 must stay on D3D11VA after an AV1 software + /// decoder has been opened and dropped. + #[test] + fn switching_h264_then_av1_then_h264_stays_alive() { + let Some(gpu) = strict_hardware_gpu("switching_h264_then_av1_then_h264_stays_alive") else { + return; + }; + assert_eq!(gpu.backend, Backend::Hardware); + let h264_path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], "switch-h264.mp4"); + let av1_path = make_legacy_av1_fixture("switch-legacy-av1.webm"); + let h264 = h264_path.to_str().expect("utf8"); + let av1 = av1_path.to_str().expect("utf8"); + + unsafe { + let mut a = Decoder::open(h264, &gpu).unwrap_or_else(|e| panic!("H.264 open: {e:#}")); + assert!(a.cpu.is_none(), "first clip must stay D3D11VA"); + first_decoded_frame(&mut a); + drop(a); + + let mut b = Decoder::open(av1, &gpu).unwrap_or_else(|e| panic!("AV1 open after H.264: {e:#}")); + assert!(b.cpu.is_some(), "AV1 clip must use CpuFrames"); + first_decoded_frame(&mut b); + drop(b); + + let mut c = Decoder::open(h264, &gpu).unwrap_or_else(|e| panic!("H.264 reopen: {e:#}")); + assert!(c.cpu.is_none(), "H.264 after AV1 must keep D3D11VA"); + first_decoded_frame(&mut c); + } + println!("CORE_ASSERTIONS_COMPLETED:switching_h264_then_av1_then_h264_stays_alive"); + } } unsafe fn drain_encoder( diff --git a/crates/compositor/src/regions.rs b/crates/compositor/src/regions.rs index 000939708..3be707066 100644 --- a/crates/compositor/src/regions.rs +++ b/crates/compositor/src/regions.rs @@ -175,9 +175,17 @@ fn lerp(a: f32, b: f32, t: f32) -> f32 { /// `startSec` (le zoom anticipe légèrement), plein régime pendant la région, ease-out après /// `endSec`. Les temps reçus sont les temps source échantillonnés par le pipeline, donc ces /// enveloppes restent alignées quand une speed region répète ou saute des frames. +/// +/// `under_trim` coupe les enveloppes : la région vit sous une coupe, donc pleine force sur son +/// span et rien en dehors. Sans ça son ease-in (1,5 s AVANT `start_sec`) et son ease-out +/// déborderaient sur les frames GARDÉES de part et d'autre du trim — un zoom que l'export ne +/// rendra jamais, visible dans la preview juste à côté de la coupe. Cf. `SceneZoomRegion`. fn zoom_region_strength(region: &SceneZoomRegion, t: f32) -> f32 { let start = region.start_sec as f32; let end = region.end_sec as f32; + if region.under_trim { + return if t >= start && t < end { 1.0 } else { 0.0 }; + } let zoom_in_end = start + ZOOM_IN_OVERLAP_S; let lead_in_start = zoom_in_end - ZOOM_IN_TRANSITION_WINDOW_S; let lead_out_end = end + TRANSITION_WINDOW_S; @@ -311,8 +319,13 @@ fn resolve_focus(region: &SceneZoomRegion, t: f32, cursor: Option<&CursorTrack>) /// transition), en secondes. Indices dans `regions` (pas d'id nécessaire — contrairement au /// web qui matche par `region.id` car il travaille sur des objets isolés, ici tout vient du /// même slice donc les positions suffisent). +/// +/// Les régions `under_trim` sont exclues du chaînage, des DEUX côtés : leur contenu est coupé au +/// rendu, donc un pan lissé vers (ou depuis) l'une d'elles ferait bouger des frames gardées au +/// nom d'une région que l'export ne joue pas. Elles restent des régions dominantes indépendantes, +/// sèches sur leur propre span (cf. `zoom_region_strength`). fn connected_pairs(regions: &[SceneZoomRegion]) -> Vec<(usize, usize, f32, f32)> { - let mut order: Vec = (0..regions.len()).collect(); + let mut order: Vec = (0..regions.len()).filter(|&i| !regions[i].under_trim).collect(); order.sort_by(|&a, &b| regions[a].start_sec.partial_cmp(®ions[b].start_sec).unwrap()); let mut pairs = Vec::new(); for w in order.windows(2) { @@ -628,6 +641,7 @@ mod zoom_focus_tests { focus_y: 0.5, focus_mode: Some("manual".into()), rotation: None, + under_trim: false, } } @@ -678,6 +692,33 @@ mod zoom_focus_tests { assert_eq!(state.scale, 1.0); assert_eq!(state.focus, [0.5, 0.5]); } + + /// Une région sous un trim est jouée SÈCHE : pleine échelle sur son span, identité juste + /// avant et juste après. `region()` couvre [2,8] et son ease-in normal démarre 1,5 s avant + /// `start_sec` — c'est exactement ce débordement qui atteindrait les frames GARDÉES autour + /// de la coupe et ferait diverger la preview de l'export. Cf. issue #216. + #[test] + fn a_region_under_a_trim_has_no_transition_window() { + let mut r = region(2.5, 0.5); + r.under_trim = true; + let regions = [r]; + assert_eq!(zoom_state_at(®ions, 1.5, None).scale, 1.0); + assert_eq!(zoom_state_at(®ions, 2.0, None).scale, 2.5); + assert_eq!(zoom_state_at(®ions, 7.9, None).scale, 2.5); + assert_eq!(zoom_state_at(®ions, 8.0, None).scale, 1.0); + } + + /// Et elle ne se chaîne pas avec sa voisine gardée : un pan lissé vers une région que + /// l'export ne joue pas ferait bouger des frames qui, elles, sont rendues. + #[test] + fn a_region_under_a_trim_is_not_chained_with_its_neighbour() { + let mut cut = region(3.0, 0.5); + cut.under_trim = true; + cut.start_sec = 9.0; + cut.end_sec = 10.0; + // Sans le filtre, l'écart de 1 s < CHAINED_ZOOM_PAN_GAP_S apparierait [2,8] et [9,10]. + assert!(connected_pairs(&[region(2.0, 0.5), cut]).is_empty()); + } } #[cfg(test)] @@ -1021,3 +1062,63 @@ mod tilt_tests { } } } + +#[cfg(test)] +mod exporter_frame_totals { + use super::*; + use crate::scene::SceneSpeedRegion; + + fn region(start_sec: f64, end_sec: f64, speed: f64) -> SceneSpeedRegion { + SceneSpeedRegion { clip_index: None, start_sec, end_sec, speed } + } + + fn frames(start_sec: f64, end_sec: f64, regions: &[SceneSpeedRegion], fps: f64) -> u64 { + speed_segments_for_window(regions, start_sec, end_sec, fps) + .iter() + .map(|segment| segment.frame_count) + .sum() + } + + /// Le total que la barre d'export doit viser, mesuré sur ce que `walk_composited_timeline` + /// itère réellement. + /// + /// Le jumeau de ce test est `src/lib/exporter/outputFrameCount.test.ts`, avec la MÊME + /// table de chiffres. Le natif n'envoie qu'un compteur de frames brut ; le total et donc + /// le pourcentage sont calculés côté TS, et rien ne reliait les deux calculs. Résultat + /// livré : le total TS ignorait les speed regions, donc un clip entièrement en 1,25× + /// rendait 80 % des frames annoncées et la barre s'arrêtait à 80 % — le « figé à ~80 % » + /// d'OpenScreen#371, au chiffre près. Toucher un côté doit faire rougir l'autre. + #[test] + fn speed_segments_match_the_exporter_frame_totals() { + const FPS: f64 = 30.0; + assert_eq!(frames(0.0, 10.0, &[], FPS), 300, "sans région"); + assert_eq!( + frames(0.0, 10.0, &[region(0.0, 10.0, 1.25)], FPS), + 240, + "1,25× : 80 % de 300, exactement le symptôme" + ); + assert_eq!(frames(0.0, 10.0, &[region(0.0, 10.0, 0.5)], FPS), 600, "0,5×"); + assert_eq!( + frames(0.0, 10.0, &[region(2.0, 4.0, 2.0)], FPS), + 60 + 30 + 180, + "couverture partielle" + ); + assert_eq!( + frames(1.0, 5.0, &[region(0.0, 100.0, 2.0)], FPS), + 60, + "région débordant la fenêtre gardée" + ); + assert_eq!( + frames(0.0, 10.0, &[region(2.0, 6.0, 2.0), region(4.0, 8.0, 4.0)], FPS), + 60 + 60 + 15 + 60, + "recouvrement : la première région garde la portion déjà couverte" + ); + assert_eq!( + frames(0.0, 10.0, &[region(0.0, 10.0, 0.0)], FPS), + 300, + "vitesse non positive traitée comme 1×" + ); + assert_eq!(frames(4.0, 4.0, &[], FPS), 0, "fenêtre vide"); + assert_eq!(frames(0.0, 10.0, &[], 0.0), 0, "fps non positif"); + } +} diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 2d20e233b..a42241672 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -331,6 +331,18 @@ pub struct SceneZoomRegion { pub focus_mode: Option, /// "iso" | "left" | "right" | null. pub rotation: Option, + /// La région entière tombe sur une portion qu'un trim retire. Ses temps sont donc HORS de + /// la fenêtre source de `clip_index`, qui n'est là que pour l'adresser (le segment que la + /// coupe interrompt, cf. `cutAddressingSegmentIndex` côté TS). + /// + /// Conséquence de rendu : la région est jouée SÈCHE, pleine force sur `[start_sec, end_sec)` + /// et rien en dehors — ni fenêtre d'ease-in/ease-out, ni chaînage avec une région voisine. + /// C'est ce qui garde la coupe : un export ne compose jamais de frame à ces temps source, + /// alors qu'une enveloppe de transition, elle, déborderait sur les frames gardées d'à côté. + /// L'utilisateur qui pose la tête de lecture sur le trim voit l'effet ; le rendu, non. + /// `#[serde(default)]` : absent de tout payload sans trim sous un modificateur (issue #216). + #[serde(default)] + pub under_trim: bool, } /// Une zone de vitesse portée par le temps source d'un clip. @@ -420,6 +432,39 @@ pub struct SceneAudio { pub gain_db: f32, } +/// One imported audio track (issue #350) mixed over the assembled programme — +/// voiceover / BGM / SFX. Deliberately a SEPARATE `Scene` field rather than a +/// member of `SceneAudio`, so `SceneAudio` stays `Copy` and the pipelines keep +/// copying it out of a borrow unchanged. +/// +/// `start_sec` is the track's head on the OUTPUT programme; `trim_start_sec` / +/// `trim_end_sec` window the source file (both source seconds). The renderer +/// resolves `start_sec` from the track's raw timeline position — equal to it when +/// the project has no trims/speed, which is the case this first cut mixes exactly. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneAudioTrack { + pub path: String, + #[serde(default)] + pub start_sec: f64, + #[serde(default)] + pub gain_db: f32, + #[serde(default)] + pub trim_start_sec: f64, + #[serde(default)] + pub trim_end_sec: Option, + /// Ramp lengths at this entry's own edges, in seconds. The app puts them only + /// on the pieces that touch the track's real start and end, so a split or + /// looping track fades once instead of at every cut or repeat. + /// + /// `#[serde(default)]` for the usual reason: a payload from a build that + /// predates the field must degrade to "no fade", not fail the whole scene. + #[serde(default)] + pub fade_in_sec: f64, + #[serde(default)] + pub fade_out_sec: f64, +} + #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneOutput { @@ -429,6 +474,43 @@ pub struct SceneOutput { pub fps: Option, } +/// Effet d'arrière-plan de la webcam. +/// +/// Ne porte que le MODE et ses paramètres — jamais des pixels. Le masque par pixel vient de +/// la segmentation qui tourne dans ce processus (`segmentation.rs`) et arrive au shader comme +/// texture `t3`. Une version antérieure faisait cuire le composite côté app et l'envoyait +/// comme piste vidéo : le codec ne sait pas porter l'alpha, et preview et export divergeaient. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneWebcamEffect { + /// "none" | "transparent" | "blur" | "custom" + pub mode: String, + /// 0..1, seulement pour `blur`. + #[serde(default)] + pub blur_intensity: f32, + /// Fond derrière le sujet pour `custom`, parsé comme `settings.wallpaper`. + #[serde(default)] + pub background: Option, + /// Chemin du modèle ONNX de segmentation. Même convention que `SceneCursorSprite::path` + /// ou qu'un wallpaper image : c'est l'app qui sait où ses assets sont installés, le + /// natif ne devine pas. Absent = pas de segmentation, l'effet reste éteint. + #[serde(default)] + pub model_path: Option, +} + +impl SceneWebcamEffect { + /// Code passé au shader dans `fx.z` : 0 = aucun (la webcam se dessine telle quelle), + /// 1 = détourage, 2 = flou, 3 = fond personnalisé. + pub(crate) fn shader_code(&self) -> f32 { + match self.mode.as_str() { + "transparent" => 1.0, + "blur" => 2.0, + "custom" => 3.0, + _ => 0.0, + } + } +} + /// Tout ce dont le natif a besoin pour composer la scène, sérialisé depuis un document. #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] @@ -451,9 +533,16 @@ pub struct Scene { /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. #[serde(default)] pub audio: SceneAudio, + /// Imported audio tracks mixed over the programme (issue #350). `#[serde(default)]`: + /// absent from every scene written before this, and from a project with none. + #[serde(default)] + pub audio_tracks: Vec, /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, + /// Effet d'arrière-plan de la webcam. Absent = aucun effet. + #[serde(default)] + pub webcam_effect: Option, /// État de rendu interne, positionné par `for_clip_window` (jamais envoyé par l'app). #[serde(skip)] pub(crate) active_clip_index: usize, @@ -469,6 +558,13 @@ impl Scene { /// Copie de scène limitée aux régions du clip actif. `clipIndex` est l'identité fiable /// lorsque plusieurs clips réutilisent les mêmes temps source ; son absence retombe sur le /// chevauchement avec la fenêtre source pour accepter les anciens payloads. + /// + /// Les deux tests étaient jusqu'ici cumulés, ce que la phrase ci-dessus ne dit pas : le + /// chevauchement est le REPLI, pas une seconde condition. La différence n'apparaît que pour + /// une région hors fenêtre, et une seule l'est — celle qui vit sous un trim (`under_trim`, + /// cf. `SceneZoomRegion`). L'app en émet une par modificateur entièrement coupé, adressée au + /// segment que la coupe interrompt, pour que la tête de lecture posée sur le trim montre ce + /// qu'il y a dessous. Exiger le chevauchement l'aurait filtrée ici même. pub(crate) fn for_clip_window( &self, clip_index: usize, @@ -477,7 +573,9 @@ impl Scene { ) -> Scene { let belongs = |region_clip_index: Option, start_sec: f64, end_sec: f64| { let overlaps_window = end_sec > source_start_sec && start_sec < source_end_sec; - overlaps_window && region_clip_index.map(|i| i == clip_index).unwrap_or(true) + region_clip_index + .map(|i| i == clip_index) + .unwrap_or(overlaps_window) }; let mut scene = self.clone(); scene.zoom_regions.retain(|region| { @@ -515,6 +613,9 @@ impl Scene { mod tests { use super::*; + /// lui. Sans ce défaut, ouvrir un projet fait par une version antérieure échouerait au + /// parse au lieu de simplement ne rien tenir (issue #560). + #[test] fn parses_a_minimal_scene_json() { let json = r##"{ @@ -601,6 +702,37 @@ mod tests { let s = Scene::from_json(json).expect("parse sans webcam_rect"); assert!(s.layout.webcam_rect.is_none()); assert_eq!(s.layout.preset, "picture-in-picture"); + assert!(s.webcam_effect.is_none()); + } + + #[test] + fn webcam_effect_maps_each_mode_to_its_shader_code() { + let scene_with = |effect: &str| { + let json = format!( + r##"{{"clips":[],"layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#000000"}},"zoomRegions":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":null}},"webcamEffect":{}}}"##, + effect + ); + Scene::from_json(&json).expect("parse avec webcamEffect").webcam_effect.expect("présent") + }; + + assert_eq!(scene_with(r#"{"mode":"none"}"#).shader_code(), 0.0); + assert_eq!(scene_with(r#"{"mode":"transparent"}"#).shader_code(), 1.0); + assert_eq!(scene_with(r#"{"mode":"blur","blurIntensity":0.75}"#).shader_code(), 2.0); + assert_eq!(scene_with(r#"{"mode":"custom"}"#).shader_code(), 3.0); + // Un mode inconnu (document trafiqué, schéma futur) ne doit pas allumer un effet. + assert_eq!(scene_with(r#"{"mode":"hologram"}"#).shader_code(), 0.0); + + let blur = scene_with(r#"{"mode":"blur","blurIntensity":0.75}"#); + assert_eq!(blur.blur_intensity, 0.75); + // `blurIntensity` absent => 0, pas une erreur de parse. + assert_eq!(scene_with(r#"{"mode":"blur"}"#).blur_intensity, 0.0); + + let custom = + scene_with(r##"{"mode":"custom","background":{"kind":"color","color":"#ff0080"}}"##); + match custom.background { + Some(SceneBackground::Color { color }) => assert_eq!(color, "#ff0080"), + other => panic!("attendu un fond couleur, obtenu {other:?}"), + } } } @@ -709,17 +841,36 @@ mod annotation_tests { #[test] fn for_clip_window_keeps_only_the_annotations_of_the_composed_clip() { - // Même règle que les zoom/speed/camera regions : bon clip ET recouvrement de la fenêtre. + // Même règle que les zoom/speed/camera regions : `clipIndex` décide seul quand il est là. + // `under-trim` porte des temps hors fenêtre EXPRÈS (il vit sous une coupe) et doit donc + // survivre : le dessin est ensuite borné par `startSec`/`endSec`, jamais atteints par un + // export. Cf. issue #216. let json = scene_json( r##"[{"id":"keep","clipIndex":0,"startSec":1.0,"endSec":2.0,"kind":"figure","x":0,"y":0,"w":0.1,"h":0.1,"zIndex":0}, {"id":"other-clip","clipIndex":1,"startSec":1.0,"endSec":2.0,"kind":"figure","x":0,"y":0,"w":0.1,"h":0.1,"zIndex":0}, - {"id":"out-of-window","clipIndex":0,"startSec":50.0,"endSec":51.0,"kind":"figure","x":0,"y":0,"w":0.1,"h":0.1,"zIndex":0}]"##, + {"id":"under-trim","clipIndex":0,"underTrim":true,"startSec":50.0,"endSec":51.0,"kind":"figure","x":0,"y":0,"w":0.1,"h":0.1,"zIndex":0}]"##, + ); + let scene = Scene::from_json(&json).expect("parse"); + let filtered = scene.for_clip_window(0, 0.0, 10.0); + assert_eq!( + filtered.annotations.iter().map(|a| a.id.as_str()).collect::>(), + vec!["keep", "under-trim"] + ); + } + + #[test] + fn for_clip_window_still_falls_back_to_window_overlap_without_a_clip_index() { + // Vieux payload : rien ne dit à quel clip la région appartient, le chevauchement de + // fenêtre reste la seule réponse disponible. C'est le REPLI, pas une seconde condition. + let json = scene_json( + r##"[{"id":"in-window","startSec":1.0,"endSec":2.0,"kind":"figure","x":0,"y":0,"w":0.1,"h":0.1,"zIndex":0}, + {"id":"out-of-window","startSec":50.0,"endSec":51.0,"kind":"figure","x":0,"y":0,"w":0.1,"h":0.1,"zIndex":0}]"##, ); let scene = Scene::from_json(&json).expect("parse"); let filtered = scene.for_clip_window(0, 0.0, 10.0); assert_eq!( filtered.annotations.iter().map(|a| a.id.as_str()).collect::>(), - vec!["keep"] + vec!["in-window"] ); } } diff --git a/crates/compositor/src/segmentation.rs b/crates/compositor/src/segmentation.rs new file mode 100644 index 000000000..ddc4dd381 --- /dev/null +++ b/crates/compositor/src/segmentation.rs @@ -0,0 +1,424 @@ +//! Segmentation du sujet webcam — le masque que `ps_main` consomme en `t3`. +//! +//! # Pourquoi l'EP CPU et pas le GPU +//! +//! Mesuré sur la cible (Radeon 610M intégré, cf. +//! `technical-documentation/engineering/webcam-segmentation.md`) : l'EP CPU coûte **+0,47 ms +//! par frame** au compositeur contre **+1,03 ms** pour DirectML, et — le point qui décide — +//! son coût **ne dépend pas de la résolution d'entrée**, là où celui de DirectML suit les +//! pixels. L'EP CPU à pleine résolution est donc moins cher que DirectML ne l'est jamais, +//! même à résolution réduite. +//! +//! Le vrai gain n'est pas la marge, il est architectural : pas de DirectML ⇒ pas de device +//! D3D12, pas de handle partagé, pas d'appariement de LUID d'adaptateur, pas de fence +//! inter-queue, et un seul chemin sur les trois plateformes au lieu de trois. +//! +//! # Le piège du nombre de threads +//! +//! Une session ONNX Runtime laissée par défaut prend tous les cœurs. Sur la machine de +//! mesure (4 cœurs) ça donne un **p95 de 24,9 ms** — une frame perdue à chaque fois que ça +//! tombe. `intra_op_num_threads = 2` est à 8 % du meilleur p10 avec moins de la moitié de la +//! traîne, et laisse deux cœurs au compositeur. La bonne valeur n'était pas la plus rapide. + +use anyhow::{bail, Result}; +use std::path::Path; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +/// Résolution d'entrée du modèle vendorisé (`selfie_segmentation_landscape.onnx`). +/// +/// Le graphe est entièrement convolutif, donc réductible — mais mesuré, ça ne sert à rien : +/// le coût de l'EP CPU est plat en résolution. Et 128x80 n'est pas livrable, la caméra en +/// plein écran agrandit le masque ~15x et les cheveux s'effondrent en rampe. +pub const MODEL_WIDTH: u32 = 256; +pub const MODEL_HEIGHT: u32 = 144; + +/// Deux threads intra-op. Voir la note du module : le défaut prend toute la machine. +const INTRA_OP_THREADS: usize = 2; + +/// La bibliothèque ONNX Runtime est-elle chargeable ? +/// +/// `ort` est lié en `load-dynamic` et **panique** quand la bibliothèque manque — +/// `load_dynamic::init(&path).expect("Failed to load ONNX Runtime dylib")`, ort/src/lib.rs. Ce +/// n'est pas une erreur qu'on peut propager : sans ce garde, un build où le staging de la lib +/// n'a pas eu lieu ferait tomber le compositeur à la première frame avec un effet, au lieu de +/// dessiner la webcam telle quelle. +#[cfg(feature = "segmentation")] +pub fn runtime_available() -> bool { + // `ORT_DYLIB_PATH` est ce que l'app pose (`ensureOnnxRuntimeOnPath`) ; sans lui, ort ira + // chercher un nom nu dans les chemins système, ce qui est le cas « pas installé ». + match std::env::var_os("ORT_DYLIB_PATH") { + Some(p) if Path::new(&p).is_file() => true, + _ => false, + } +} + +#[cfg(not(feature = "segmentation"))] +pub fn runtime_available() -> bool { + false +} + +/// Segmenteur chargé, prêt à produire un masque par frame. +pub struct Segmenter { + #[cfg(feature = "segmentation")] + session: ort::session::Session, + /// Réutilisé d'une frame à l'autre pour ne pas réallouer 110 Ko à 30 Hz. + input_scratch: Vec, + mask_scratch: Vec, +} + +impl Segmenter { + /// Charge le modèle ONNX. `model_path` est le `.onnx` vendorisé à côté des `.tflite`. + #[cfg(feature = "segmentation")] + pub fn load(model_path: &Path) -> Result { + if !model_path.exists() { + bail!("modèle de segmentation absent : {}", model_path.display()); + } + if !runtime_available() { + bail!( + "bibliothèque ONNX Runtime introuvable (ORT_DYLIB_PATH={:?}) — l'effet reste éteint", + std::env::var_os("ORT_DYLIB_PATH") + ); + } + // `ort::Error` est générique sur le type du builder, donc il ne satisfait pas les + // bornes d'`anyhow::Context` — d'où le `map_err` explicite plutôt qu'un `?` direct. + // Deuxième garde, pour le cas « le fichier est là mais ne se charge pas » (mauvaise + // architecture, dépendance manquante) : ort panique là aussi, et une panique qui + // traverse le thread de rendu tue la preview. + let session = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + (|| -> ort::Result { + ort::session::Session::builder()? + .with_intra_threads(INTRA_OP_THREADS)? + // Un seul thread inter-op : le graphe est une chaîne, il n'y a rien à + // paralléliser entre branches, et un pool de plus ne ferait que disputer les + // cœurs au compositeur. + .with_inter_threads(1)? + .commit_from_file(model_path) + })() + })) + .map_err(|_| anyhow::anyhow!("ONNX Runtime a paniqué au chargement — effet désactivé"))? + .map_err(|e| anyhow::anyhow!("chargement de {} : {e}", model_path.display()))?; + Ok(Self { + session, + input_scratch: vec![0.0; (MODEL_WIDTH * MODEL_HEIGHT * 3) as usize], + mask_scratch: vec![0; (MODEL_WIDTH * MODEL_HEIGHT) as usize], + }) + } + + #[cfg(not(feature = "segmentation"))] + pub fn load(_model_path: &Path) -> Result { + bail!("compilé sans la feature `segmentation`") + } + + /// Produit le masque du sujet à partir d'une frame RGB8 déjà mise à l'échelle du modèle. + /// + /// `rgb` fait `MODEL_WIDTH * MODEL_HEIGHT * 3` octets, entrelacé R,G,B. Le retour fait + /// `MODEL_WIDTH * MODEL_HEIGHT` octets, 0 = fond, 255 = sujet — exactement ce que + /// `Compositor::set_webcam_mask` attend. + /// + /// Le redimensionnement n'est pas fait ici : l'appelant a déjà la frame sur le GPU et sait + /// la réduire bien mieux qu'une boucle CPU. + #[cfg(feature = "segmentation")] + pub fn run(&mut self, rgb: &[u8]) -> Result<&[u8]> { + let expected = (MODEL_WIDTH * MODEL_HEIGHT * 3) as usize; + if rgb.len() != expected { + bail!("frame de {} octets, {expected} attendus", rgb.len()); + } + // Le modèle veut du 0..1 en NHWC — le même ordre que la frame entrelacée, donc une + // simple division sans transposition. + for (dst, &src) in self.input_scratch.iter_mut().zip(rgb.iter()) { + *dst = src as f32 * (1.0 / 255.0); + } + + // `TensorRef` emprunte le scratch au lieu de le copier : à 30 Hz, 442 Ko recopiés par + // frame pour rien seraient exactement le genre de coût que cette conception évite. + let shape = [1_i64, MODEL_HEIGHT as i64, MODEL_WIDTH as i64, 3]; + let input = ort::value::TensorRef::from_array_view((shape, self.input_scratch.as_slice())) + .map_err(|e| anyhow::anyhow!("construction du tenseur d'entrée : {e}"))?; + let outputs = self + .session + .run(ort::inputs!["input_1" => input]) + .map_err(|e| anyhow::anyhow!("inférence : {e}"))?; + let (_, mask) = outputs["segment_back"] + .try_extract_tensor::() + .map_err(|e| anyhow::anyhow!("extraction du masque : {e}"))?; + + if mask.len() != self.mask_scratch.len() { + bail!("masque de {} valeurs, {} attendues", mask.len(), self.mask_scratch.len()); + } + // Déjà passé par une sigmoïde dans le graphe, donc borné 0..1 — le clamp ne protège + // que d'un modèle regénéré différemment. + for (dst, &src) in self.mask_scratch.iter_mut().zip(mask.iter()) { + *dst = (src.clamp(0.0, 1.0) * 255.0) as u8; + } + Ok(&self.mask_scratch) + } + + #[cfg(not(feature = "segmentation"))] + pub fn run(&mut self, _rgb: &[u8]) -> Result<&[u8]> { + bail!("compilé sans la feature `segmentation`") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Le chemin par défaut du modèle vendorisé, depuis la racine du dépôt. + fn vendored_model() -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx") + } + + #[test] + fn the_vendored_model_is_where_the_loader_expects_it() { + // Ne charge pas le modèle (la feature peut être éteinte) : vérifie seulement que le + // fichier que `load` ira chercher existe et n'est pas un pointeur LFS ou un tronçon. + let path = vendored_model(); + let meta = std::fs::metadata(&path) + .unwrap_or_else(|e| panic!("modèle introuvable en {} : {e}", path.display())); + assert!(meta.len() > 100_000, "modèle suspicieusement petit : {} octets", meta.len()); + } + + + #[cfg(feature = "segmentation")] + #[test] + fn a_missing_model_is_refused_before_the_runtime_is_even_touched() { + // Ce test-ci tourne PARTOUT : le chemin est vérifié avant tout appel à ort, ce qui + // est précisément la garantie qu'on veut (pas de panique sur une machine sans lib). + let err = match Segmenter::load(Path::new("nexiste/pas.onnx")) { + Ok(_) => panic!("un modèle inexistant ne doit pas charger"), + Err(e) => e.to_string(), + }; + assert!(err.contains("nexiste"), "message peu utile : {err}"); + } + + #[cfg(feature = "segmentation")] + #[test] + fn a_missing_runtime_is_an_error_not_a_panic() { + if runtime_available() { + eprintln!("ONNX Runtime présent — le cas « absent » n'est pas exerçable ici"); + return; + } + // Le modèle EXISTE, donc on va bien jusqu'au garde du runtime. Sans lui, ort + // paniquerait et emporterait le thread de rendu. + match Segmenter::load(&vendored_model()) { + Ok(_) => panic!("chargement réussi sans bibliothèque ?"), + Err(e) => assert!( + e.to_string().contains("ONNX Runtime"), + "l'erreur doit nommer la bibliothèque manquante : {e}" + ), + } + } + + /// Les tests qui font tourner une vraie inférence n'ont de sens que là où la bibliothèque + /// est installée. La CI macOS et Linux ne la stage pas encore, et un test rouge pour ça + /// dirait quelque chose de faux sur le code. + #[cfg(feature = "segmentation")] + fn skip_without_runtime() -> bool { + if runtime_available() { + return false; + } + eprintln!("ONNX Runtime absent (ORT_DYLIB_PATH non posé) — test sauté"); + true + } + + #[cfg(feature = "segmentation")] + #[test] + fn a_frame_of_the_wrong_size_is_refused_rather_than_read_out_of_bounds() { + if skip_without_runtime() { + return; + } + let mut seg = Segmenter::load(&vendored_model()).expect("chargement du modèle"); + let err = seg.run(&[0u8; 12]).unwrap_err().to_string(); + assert!(err.contains("attendus"), "message peu utile : {err}"); + } + + #[test] + fn the_rate_limiter_admits_one_frame_per_interval() { + let mut rl = RateLimiter::new(30); + let t0 = Instant::now(); + assert!(rl.should_run(t0), "la première frame passe toujours"); + assert!(!rl.should_run(t0 + Duration::from_millis(10)), "10 ms < 33 ms"); + assert!(!rl.should_run(t0 + Duration::from_millis(33)), "juste sous l'intervalle"); + assert!(rl.should_run(t0 + Duration::from_millis(34)), "au-delà de l'intervalle"); + // Le pas repart du dernier passage accepté, pas du premier : sinon la cadence + // dériverait vers le haut après chaque frame refusée. + assert!(!rl.should_run(t0 + Duration::from_millis(40))); + assert!(rl.should_run(t0 + Duration::from_millis(68))); + } + + #[test] + fn a_60_hz_render_loop_yields_about_30_inferences_per_second() { + let mut rl = RateLimiter::new(30); + let t0 = Instant::now(); + let admitted = (0..60) + .filter(|i| rl.should_run(t0 + Duration::from_micros(16_667 * i))) + .count(); + assert_eq!(admitted, 30, "60 frames rendues doivent donner 30 inférences"); + } + + #[cfg(feature = "segmentation")] + #[test] + fn the_worker_drops_stale_frames_rather_than_queueing_them() { + if skip_without_runtime() { + return; + } + use std::sync::atomic::{AtomicUsize, Ordering}; + + let seen = Arc::new(AtomicUsize::new(0)); + let counter = Arc::clone(&seen); + let worker = SegmentationWorker::spawn( + Segmenter::load(&vendored_model()).expect("chargement du modèle"), + move |mask, w, h| { + assert_eq!(mask.len(), (w * h) as usize); + counter.fetch_add(1, Ordering::SeqCst); + }, + ); + + // Cent frames déposées d'affilée : le worker en traite bien moins que cent, puisque + // chaque dépôt écrase le précédent non consommé. La borne est large — le test pin le + // fait qu'on écrase, pas un débit. + let frame = vec![90u8; (MODEL_WIDTH * MODEL_HEIGHT * 3) as usize]; + for _ in 0..100 { + worker.submit(&frame); + } + std::thread::sleep(Duration::from_millis(300)); + let done = seen.load(Ordering::SeqCst); + assert!(done > 0, "le worker n'a rien traité"); + assert!(done < 100, "{done} inférences pour 100 dépôts : la file s'accumule"); + } + + #[cfg(feature = "segmentation")] + #[test] + fn segments_a_uniform_frame_without_panicking_and_returns_the_right_size() { + if skip_without_runtime() { + return; + } + let mut seg = Segmenter::load(&vendored_model()).expect("chargement du modèle"); + let frame = vec![128u8; (MODEL_WIDTH * MODEL_HEIGHT * 3) as usize]; + let mask = seg.run(&frame).expect("inférence"); + assert_eq!(mask.len(), (MODEL_WIDTH * MODEL_HEIGHT) as usize); + // Un gris uniforme ne contient pas de sujet : le masque doit être massivement du + // fond. C'est une borne large, pas une assertion de qualité — elle attrape un modèle + // qui renverrait du bruit ou du plein. + let subject = mask.iter().filter(|&&v| v > 128).count(); + assert!( + subject * 10 < mask.len(), + "{subject} pixels sujet sur {} pour une image unie", + mask.len() + ); + } +} + +/// Cadence l'inférence : 30 Hz, pas la fréquence de rendu. +/// +/// Une silhouette ne change pas de façon perceptible en 16 ms, et c'est le seul levier +/// mesuré qui divise le coût par deux sans toucher au modèle ni à sa précision. Le chemin +/// export tourne déjà à 30 Hz. +pub struct RateLimiter { + interval: Duration, + last: Option, +} + +impl RateLimiter { + pub fn new(hz: u32) -> Self { + Self { interval: Duration::from_secs_f64(1.0 / hz.max(1) as f64), last: None } + } + + /// `true` si assez de temps s'est écoulé depuis le dernier passage. Prend `now` en + /// paramètre plutôt que de lire l'horloge : c'est ce qui rend la cadence testable. + pub fn should_run(&mut self, now: Instant) -> bool { + match self.last { + Some(prev) if now.duration_since(prev) < self.interval => false, + _ => { + self.last = Some(now); + true + } + } + } +} + +/// Boîte d'échange à une place, qui écrase au lieu d'empiler. +/// +/// Si l'inférence prend du retard, la bonne réponse est de sauter des frames, pas d'en +/// accumuler : un masque en retard de trois frames est pire qu'un masque sauté, et une file +/// qui grandit finit par manger la mémoire. `submit` remplace donc silencieusement une frame +/// non consommée. +struct Slot { + frame: Mutex>>, + ready: Condvar, + stop: Mutex, +} + +/// Thread d'inférence : reçoit des frames RGB, publie des masques via un callback. +/// +/// Le callback est appelé depuis le thread du worker, pas depuis celui du rendu — c'est +/// `Compositor::set_webcam_mask` qui est prévu pour ça, le device étant multithread-protected. +pub struct SegmentationWorker { + slot: Arc, + handle: Option>, +} + +impl SegmentationWorker { + /// Démarre le worker. `on_mask` reçoit le masque et ses dimensions à chaque inférence. + pub fn spawn( + mut segmenter: Segmenter, + on_mask: impl Fn(&[u8], u32, u32) + Send + 'static, + ) -> Self { + let slot = Arc::new(Slot { + frame: Mutex::new(None), + ready: Condvar::new(), + stop: Mutex::new(false), + }); + let worker_slot = Arc::clone(&slot); + let handle = std::thread::Builder::new() + .name("openscreen-segmentation".into()) + .spawn(move || loop { + let frame = { + let mut guard = worker_slot.frame.lock().unwrap(); + while guard.is_none() { + if *worker_slot.stop.lock().unwrap() { + return; + } + let (g, timeout) = worker_slot + .ready + .wait_timeout(guard, Duration::from_millis(100)) + .unwrap(); + guard = g; + if timeout.timed_out() && guard.is_none() { + if *worker_slot.stop.lock().unwrap() { + return; + } + } + } + guard.take().expect("non vide, la boucle vient de le vérifier") + }; + match segmenter.run(&frame) { + Ok(mask) => on_mask(mask, MODEL_WIDTH, MODEL_HEIGHT), + // Une frame ratée est sautée, pas fatale : le masque précédent reste + // affiché, ce qui vaut mieux qu'un effet qui clignote. + Err(e) => eprintln!("[segmentation] frame ignorée : {e}"), + } + }) + .expect("le thread de segmentation doit démarrer"); + Self { slot, handle: Some(handle) } + } + + /// Dépose une frame à segmenter. Écrase celle qui attendait, s'il y en avait une. + pub fn submit(&self, rgb: &[u8]) { + let mut guard = self.slot.frame.lock().unwrap(); + *guard = Some(rgb.to_vec()); + self.slot.ready.notify_one(); + } +} + +impl Drop for SegmentationWorker { + fn drop(&mut self) { + *self.slot.stop.lock().unwrap() = true; + self.slot.ready.notify_all(); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} diff --git a/crates/compositor/src/shaders.hlsl b/crates/compositor/src/shaders.hlsl index b9d438d7d..d9b6a13ac 100644 --- a/crates/compositor/src/shaders.hlsl +++ b/crates/compositor/src/shaders.hlsl @@ -39,6 +39,10 @@ VSOut vs_main(uint vid : SV_VertexID) Texture2D texY : register(t0); Texture2D texUV : register(t1); Texture2D texImg : register(t2); // wallpaper image RGBA (fond, mode 6) +// Masque de segmentation du sujet, 0 = fond, 1 = sujet. Produit par `segmentation.rs` a la +// resolution du modele (256x144) ; l'upscale vers la resolution webcam est fait par le sampler +// lineaire, ce qui est exactement le filtrage qu'on veut sur un masque. +Texture2D texMask : register(t3); SamplerState samp : register(s0); // BT.709 limited -> RGB (§7 E1), matrice en dur, range mesuré en S1. @@ -78,6 +82,22 @@ float sd_round_rect(float2 p, float2 halfsz, float r) return length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - r; } +// Couverture du quad avec coins arrondis, pour les modes qui retournent AVANT la queue de +// `ps_main` (5 gradient, 6 image). Ils s'en passaient tant qu'ils ne servaient qu'au fond plein +// cadre, qui n'a pas de rayon ; depuis que la bulle webcam peut porter un dégradé ou une image, +// sans ça le fond déborde en carré opaque sur les coins arrondis de la bulle et mange l'ombre. +// Renvoie 1.0 quand aucun rayon n'est demandé — le fond plein cadre est donc inchangé. +float quad_round_alpha(float2 local, float2 quad_px, float radius_px) +{ + if (radius_px <= 0.0 || quad_px.x <= 0.0 || quad_px.y <= 0.0) + { + return 1.0; + } + float2 halfsz = quad_px * 0.5; + float d = sd_round_rect(local - halfsz, halfsz, radius_px); + return 1.0 - smoothstep(0.0, 1.5, d); // même feather ~1.5px que la queue +} + // Intersection de deux droites données par (normale, offset) : n·x = d. Cramer. float2 line_cross(float2 n1, float d1, float2 n2, float d2) { @@ -154,6 +174,55 @@ float3 quad_inverse_bilinear(float2 P, float2 c00, float2 c10, float2 c11, float return (r0.z > 0.5) ? r0 : r1; } +// Fond flouté pour le mode "blur" de la webcam. +// Disque de Vogel (spirale à angle d'or) à 21 échantillons avec pondération gaussienne et +// rotation par pixel via Interleaved Gradient Noise (IGN) pour un bokeh photographique doux, isotrope et rapide. +static const float3 VOGEL_TAPS[21] = { + float3( 0.154303, 0.000000, 0.942213), + float3(-0.197070, 0.180532, 0.836464), + float3( 0.030165, -0.343712, 0.742584), + float3( 0.248394, 0.323986, 0.659241), + float3(-0.455834, -0.080631, 0.585251), + float3( 0.431806, -0.274679, 0.519566), + float3(-0.144431, 0.537274, 0.461253), + float3(-0.275445, -0.530352, 0.409484), + float3( 0.597605, 0.218244, 0.363526), + float3(-0.621708, 0.256632, 0.322726), + float3( 0.299704, -0.640451, 0.286505), + float3( 0.221474, 0.706094, 0.254349), + float3(-0.667525, -0.386844, 0.225802), + float3( 0.783083, -0.172159, 0.200460), + float3(-0.477903, 0.679768, 0.177961), + float3(-0.110407, -0.852001, 0.157988), + float3( 0.677789, 0.571241, 0.140256), + float3(-0.912091, 0.037718, 0.124514), + float3( 0.665301, -0.662063, 0.110540), + float3(-0.044511, 0.962596, 0.098133), + float3(-0.633036, -0.758588, 0.087119) +}; + +float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px) +{ + float max_r_px = max(intensity, 0.0) * 22.0 + 1.5; + float2 step = max_r_px / max(qpx, 1.0); + // Interleaved Gradient Noise pour rotation aléatoire par pixel + float noise = frac(52.9829189 * frac(0.06711056 * local_px.x + 0.00583715 * local_px.y)); + float angle = noise * 6.2831853; + float s, c; + sincos(angle, s, c); + float3 sum = 0.0; + float total = 0.0; + [unroll] for (int k = 0; k < 21; k++) + { + float2 p = VOGEL_TAPS[k].xy; + float w = VOGEL_TAPS[k].z; + float2 rot_p = float2(p.x * c - p.y * s, p.x * s + p.y * c); + sum += sample_yuv(saturate(uv + rot_p * step)) * w; + total += w; + } + return sum / max(total, 1e-4); +} + float4 ps_main(VSOut i) : SV_Target { // mode 13 : SPRITE DE CURSEUR posé sur l'écran incliné. Même warp que le mode 8, mais @@ -352,7 +421,8 @@ float4 ps_main(VSOut i) : SV_Target // recouvrement), i.uv l'interpole. Opaque. if (mode > 5.5) { - return float4(texImg.Sample(samp, i.uv).rgb, 1.0); + float a = quad_round_alpha(i.local, quad_px, radius_px); + return float4(texImg.Sample(samp, i.uv).rgb * a, a); // prémultiplié } // mode 5 : gradient linéaire 2 stops (parité web wallpaper dégradé). color = stop0, @@ -362,9 +432,15 @@ float4 ps_main(VSOut i) : SV_Target { float2 dir = fx.xy; float denom = max(abs(dir.x) + abs(dir.y), 1e-4); - float t = saturate(0.5 + dot(i.pout - 0.5, dir) / denom); + // Paramétré sur le QUAD dès qu'il en a un (la bulle webcam), sinon sur la sortie. Pour le + // fond plein cadre les deux coïncident ; pour une bulle dans un coin, `pout` ne montrerait + // que la tranche du dégradé plein cadre qui passe dessous, jamais la rampe complète que + // le sélecteur affiche. + float2 gp = (quad_px.x > 0.0 && quad_px.y > 0.0) ? (i.local / quad_px) : i.pout; + float t = saturate(0.5 + dot(gp - 0.5, dir) / denom); float3 g = lerp(color.rgb, src.xyz, t); - return float4(g, 1.0); // opaque, prémultiplié (a=1) + float a = quad_round_alpha(i.local, quad_px, radius_px); + return float4(g * a, a); // prémultiplié } // mode 4 : curseur custom (dot + ring, dessiné depuis les maths). color = teinte. @@ -410,6 +486,8 @@ float4 ps_main(VSOut i) : SV_Target } float3 rgb; + // 1 sauf en mode detourage, ou il porte le masque du sujet (cf. la branche fx.z ci-dessous). + float alpha_mask = 1.0; if (mode < 0.5) { // flou de mouvement par vélocité (§8) : pour CE pixel sortie, uv à la frame @@ -419,8 +497,10 @@ float4 ps_main(VSOut i) : SV_Target float2 localp = (i.pout - dst_prev.xy) / dst_prev.zw; float2 uv_prev = src_prev.xy + localp * (src_prev.zw - src_prev.xy); float2 duv = uv_now - uv_prev; + float mb_scale = saturate(mb.y); + float2 duv_blur = duv * mb_scale; int taps = (int) mb.x; - if (taps <= 1 || dot(duv, duv) < 1e-9) + if (taps <= 1 || mb_scale <= 0.001 || dot(duv_blur, duv_blur) < 1e-9) { rgb = sample_yuv(uv_now); } @@ -431,17 +511,45 @@ float4 ps_main(VSOut i) : SV_Target { if (k >= taps) break; float t = (float) k / (float) (taps - 1); - acc += sample_yuv(uv_prev + duv * t); + acc += sample_yuv(uv_now - duv_blur * (1.0 - t)); } rgb = acc / (float) taps; } + + // Effet d'arriere-plan webcam. fx.z : 1 = detourage, 2 = flou, 3 = fond personnalise. + // `color` porte la couleur de fond du mode 3, fx.w l'intensite du flou du mode 2. + // fx.xy porte l'etendue VALIDE de la texture webcam (wcw/wtw, wch/wth) : le masque a + // ete produit sur la frame ENTIERE, pas sur le sous-rect dessine, pour que le modele + // ne se fasse pas amputer le sujet par un crop utilisateur. Il faut donc ramener uv, + // qui vit dans l'espace source, dans cet espace-la. + // Le masque est absent (texture 1x1 noire) tant que la segmentation n'a pas produit sa + // premiere frame : `person` vaut alors 0 et le mode 1 rendrait la webcam invisible, donc + // c'est l'appelant qui ne met fx.z a autre chose que 0 qu'une fois un masque disponible. + float effect = fx.z; + if (effect > 0.5) + { + float2 mask_uv = uv_now / max(fx.xy, 1e-6); + float person = saturate(texMask.Sample(samp, mask_uv)); + if (effect > 2.5) + { + rgb = lerp(color.rgb, rgb, person); + } + else if (effect > 1.5) + { + rgb = lerp(blur_webcam_bg(uv_now, fx.w, quad_px, i.local), rgb, person); + } + else + { + alpha_mask = person; + } + } } else { rgb = color.rgb; } - float alpha = color.a; + float alpha = color.a * alpha_mask; if (radius_px > 0.0) { // `quad_px` est en px de SORTIE (le render target porte la géométrie de sortie) et diff --git a/crates/compositor/src/shaders.metal b/crates/compositor/src/shaders.metal index 64dd8d476..9c89eca70 100644 --- a/crates/compositor/src/shaders.metal +++ b/crates/compositor/src/shaders.metal @@ -151,6 +151,22 @@ inline float sd_round_rect(float2 p, float2 halfsz, float r) return length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - r; } +// Couverture du quad avec coins arrondis, pour les modes qui retournent AVANT la queue de +// `ps_main` (5 gradient, 6 image). Ils s'en passaient tant qu'ils ne servaient qu'au fond plein +// cadre, qui n'a pas de rayon ; depuis que la bulle webcam peut porter un dégradé ou une image, +// sans ça le fond déborde en carré opaque sur les coins arrondis de la bulle et mange l'ombre. +// Renvoie 1.0 quand aucun rayon n'est demandé — le fond plein cadre est donc inchangé. +inline float quad_round_alpha(float2 local, float2 quad_px, float radius_px) +{ + if (radius_px <= 0.0 || quad_px.x <= 0.0 || quad_px.y <= 0.0) + { + return 1.0; + } + float2 halfsz = quad_px * 0.5; + float d = sd_round_rect(local - halfsz, halfsz, radius_px); + return 1.0 - smoothstep(0.0, 1.5, d); // même feather ~1.5px que la queue +} + // Intersection de deux droites données par (normale, offset) : n·x = d. Cramer. inline float2 line_cross(float2 n1, float d1, float2 n2, float d2) { @@ -219,11 +235,67 @@ inline float3 quad_inverse_bilinear(float2 P, float2 c00, float2 c10, float2 c11 // Identique à `ps_main` côté HLSL ligne pour ligne (à la syntaxe MSL près). // ================================================================================= +// Fond floute du mode "blur" webcam. Miroir de `blur_webcam_bg` cote HLSL : memes 25 taps, +// memes poids, meme rayon — les deux back-ends doivent rendre le meme pixel. +// Fond flouté pour le mode "blur" de la webcam. +// Disque de Vogel (spirale à angle d'or) à 21 échantillons avec pondération gaussienne et +// rotation par pixel via Interleaved Gradient Noise (IGN) pour un bokeh photographique doux, isotrope et rapide. +constant float3 VOGEL_TAPS[21] = { + float3( 0.154303, 0.000000, 0.942213), + float3(-0.197070, 0.180532, 0.836464), + float3( 0.030165, -0.343712, 0.742584), + float3( 0.248394, 0.323986, 0.659241), + float3(-0.455834, -0.080631, 0.585251), + float3( 0.431806, -0.274679, 0.519566), + float3(-0.144431, 0.537274, 0.461253), + float3(-0.275445, -0.530352, 0.409484), + float3( 0.597605, 0.218244, 0.363526), + float3(-0.621708, 0.256632, 0.322726), + float3( 0.299704, -0.640451, 0.286505), + float3( 0.221474, 0.706094, 0.254349), + float3(-0.667525, -0.386844, 0.225802), + float3( 0.783083, -0.172159, 0.200460), + float3(-0.477903, 0.679768, 0.177961), + float3(-0.110407, -0.852001, 0.157988), + float3( 0.677789, 0.571241, 0.140256), + float3(-0.912091, 0.037718, 0.124514), + float3( 0.665301, -0.662063, 0.110540), + float3(-0.044511, 0.962596, 0.098133), + float3(-0.633036, -0.758588, 0.087119) +}; + +inline float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px, + texture2d texY, + texture2d texUV) +{ + float max_r_px = max(intensity, 0.0) * 22.0 + 1.5; + float2 step = max_r_px / max(qpx, float2(1.0)); + float noise = fract(52.9829189 * fract(0.06711056 * local_px.x + 0.00583715 * local_px.y)); + float angle = noise * 6.2831853; + float s = sin(angle); + float c = cos(angle); + float3 sum = float3(0.0); + float total = 0.0; + for (int k = 0; k < 21; k++) + { + float2 p = VOGEL_TAPS[k].xy; + float w = VOGEL_TAPS[k].z; + float2 rot_p = float2(p.x * c - p.y * s, p.x * s + p.y * c); + sum += sample_yuv(saturate(uv + rot_p * step), texY, texUV) * w; + total += w; + } + return sum / max(total, 1e-4); +} + fragment float4 ps_main(VSOut i [[stage_in]], constant Layer &layer [[buffer(0)]], texture2d texY [[texture(0)]], texture2d texUV [[texture(1)]], - texture2d texImg [[texture(2)]]) + texture2d texImg [[texture(2)]], + // Masque de segmentation du sujet webcam. Non lie tant qu'aucun + // masque n'existe : Metal rend alors 0, ce qui est sans effet + // puisque la branche n'est prise que si layer.fx.z > 0.5. + texture2d texMask [[texture(3)]]) { // mode 13 : SPRITE DE CURSEUR posé sur l'écran incliné. Cf. commentaires HLSL. if (layer.mode > 12.5) @@ -344,7 +416,8 @@ fragment float4 ps_main(VSOut i [[stage_in]], // « le wallpaper est dessiné avec alpha 0 ». if (layer.mode > 5.5 && layer.mode < 6.5) { - return float4(texImg.sample(samp, i.uv).rgb, 1.0); + float a = quad_round_alpha(i.local, layer.quad_px, layer.radius_px); + return float4(texImg.sample(samp, i.uv).rgb * a, a); // prémultiplié } // mode 5 : gradient linéaire 2 stops (parité web wallpaper dégradé). color = stop0, @@ -357,9 +430,17 @@ fragment float4 ps_main(VSOut i [[stage_in]], { float2 dir = layer.fx.xy; float denom = max(abs(dir.x) + abs(dir.y), 1e-4); - float t = clamp(0.5 + dot(i.pout - 0.5, dir) / denom, 0.0, 1.0); + // Paramétré sur le QUAD dès qu'il en a un (la bulle webcam), sinon sur la sortie. Pour le + // fond plein cadre les deux coïncident ; pour une bulle dans un coin, `pout` ne montrerait + // que la tranche du dégradé plein cadre qui passe dessous, jamais la rampe complète que + // le sélecteur affiche. + float2 gp = (layer.quad_px.x > 0.0 && layer.quad_px.y > 0.0) + ? (i.local / layer.quad_px) + : i.pout; + float t = clamp(0.5 + dot(gp - 0.5, dir) / denom, 0.0, 1.0); float3 g = mix(layer.color.rgb, layer.src.xyz, t); - return float4(g, 1.0); // opaque, prémultiplié (a=1) + float a = quad_round_alpha(i.local, layer.quad_px, layer.radius_px); + return float4(g * a, a); // prémultiplié } // mode 4 : curseur dessiné (dot + ring SDF). @@ -467,6 +548,8 @@ fragment float4 ps_main(VSOut i [[stage_in]], } float3 rgb; + // 1 sauf en detourage, ou il porte le masque du sujet. Cf. la branche fx.z plus bas. + float alpha_mask = 1.0; if (layer.mode < 0.5) { // flou de mouvement par vélocité (§8) @@ -474,8 +557,10 @@ fragment float4 ps_main(VSOut i [[stage_in]], float2 localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; float2 uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); float2 duv = uv_now - uv_prev; + float mb_scale = saturate(layer.mb.y); + float2 duv_blur = duv * mb_scale; int taps = int(layer.mb.x); - if (taps <= 1 || dot(duv, duv) < 1e-9) + if (taps <= 1 || mb_scale <= 0.001 || dot(duv_blur, duv_blur) < 1e-9) { rgb = sample_yuv(uv_now, texY, texUV); } @@ -486,17 +571,39 @@ fragment float4 ps_main(VSOut i [[stage_in]], { if (k >= taps) break; float t = float(k) / float(taps - 1); - acc += sample_yuv(uv_prev + duv * t, texY, texUV); + acc += sample_yuv(uv_now - duv_blur * (1.0 - t), texY, texUV); } rgb = acc / float(taps); } + + // Effet d'arriere-plan webcam. Miroir exact de la branche HLSL : fx.z porte le mode + // (1 = detourage, 2 = flou, 3 = fond plat), fx.w l'intensite du flou, fx.xy l'etendue + // valide de la texture webcam pour ramener uv dans l'espace du masque. + float effect = layer.fx.z; + if (effect > 0.5) + { + float2 mask_uv = uv_now / max(layer.fx.xy, float2(1e-6)); + float person = saturate(texMask.sample(samp, mask_uv).r); + if (effect > 2.5) + { + rgb = mix(layer.color.rgb, rgb, person); + } + else if (effect > 1.5) + { + rgb = mix(blur_webcam_bg(uv_now, layer.fx.w, layer.quad_px, i.local, texY, texUV), rgb, person); + } + else + { + alpha_mask = person; + } + } } else { rgb = layer.color.rgb; } - float alpha = layer.color.a; + float alpha = layer.color.a * alpha_mask; if (layer.radius_px > 0.0) { float2 halfsz = layer.quad_px * 0.5; diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index 721fe7902..fe3a2e795 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -17,6 +17,7 @@ use crate::compositor::Compositor; use crate::config::Cfg; use crate::cursor::CursorTrack; use crate::d3d::Gpu; +use crate::ffi::AVFrame; use crate::frame_geometry::webcam_is_real; use crate::pipeline::{ClipSource, Decoder}; use crate::regions::{speed_segments_for_window, SpeedSegment}; @@ -164,6 +165,14 @@ pub(crate) unsafe fn walk_composited_timeline( let mut frames: u64 = 0; + // L'export doit être reproductible : deux rendus du même projet, les mêmes pixels. Cette + // boucle avance aussi vite que la machine décode, sans rapport avec le temps réel, alors que + // la segmentation est cadencée à l'horloge et calculée sur un worker — deux choix faits pour + // la preview, et qui deviennent ici des bugs : le nombre de frames couvertes par un masque + // suivrait la charge machine, et les premières frames sortiraient AVANT le premier masque, + // donc avec le vrai arrière-plan de la webcam gravé dans le fichier. + comp.set_segmentation_deterministic(true); + for (clip_index, clip) in clips.iter().enumerate() { // Le preset de layout est GLOBAL (un seul panneau pour toute la timeline) mais la // caméra est PAR CLIP : un projet mélange sans problème un enregistrement avec webcam @@ -194,10 +203,10 @@ pub(crate) unsafe fn walk_composited_timeline( comp.set_has_webcam(has_camera); let webcam_key = if has_camera { &clip.webcam } else { &clip.screen }; if !screen_decs.contains_key(&clip.screen) { - screen_decs.insert(clip.screen.clone(), Decoder::open(&clip.screen, gpu)?); + screen_decs.insert(clip.screen.clone(), Decoder::open_for_export(&clip.screen, gpu)?); } if !webcam_decs.contains_key(webcam_key) { - webcam_decs.insert(webcam_key.clone(), Decoder::open(webcam_key, gpu)?); + webcam_decs.insert(webcam_key.clone(), Decoder::open_for_export(webcam_key, gpu)?); } let sdec = screen_decs.get_mut(&clip.screen).unwrap(); let wdec = webcam_decs.get_mut(webcam_key).unwrap(); @@ -297,11 +306,17 @@ pub(crate) unsafe fn walk_composited_timeline( for segment_frame in 0..segment.frame_count { let target_source_time = segment.start_sec + segment_frame as f64 * segment.speed / out_fps as f64; - if !advance_decoder_to(sdec, target_source_time, 0.0)? { - break 'clip_frames; + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::DecodeScreen); + if !advance_decoder_to(sdec, target_source_time, 0.0)? { + break 'clip_frames; + } } - if !advance_decoder_to(wdec, target_source_time, clip.webcam_offset_sec)? { - break 'clip_frames; + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::DecodeWebcam); + if !advance_decoder_to(wdec, target_source_time, clip.webcam_offset_sec)? { + break 'clip_frames; + } } let sf = sdec.cur_frame(); let wf = wdec.cur_frame(); @@ -313,12 +328,16 @@ pub(crate) unsafe fn walk_composited_timeline( if cursor_enabled && cursor_active_path.is_some() { comp.set_cursor_time(Some(target_source_time as f32)); } - comp.compose_frame(sf, wf, frames as f32, cfg)?; + { + let _p = crate::export_probe::scope(crate::export_probe::Stage::Compose); + comp.compose_frame(sf, wf, frames as f32, cfg)?; + } on_frame(frames)?; frames += 1; } } + on_clip_end( clip_index, source_end_sec, @@ -329,6 +348,8 @@ pub(crate) unsafe fn walk_composited_timeline( comp.set_cursor_time(None); comp.set_timeline_time(None); + // Le compositeur est réutilisé par la preview après un export : lui rendre sa cadence. + comp.set_segmentation_deterministic(false); Ok(frames) } diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 6fb2a73ed..fd7da966d 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -32,8 +32,15 @@ struct Layer { @group(0) @binding(0) var layer: Layer; @group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r -@group(0) @binding(2) var texUV: texture_2d; // Rg8Unorm, sample .rg +@group(0) @binding(2) var texU: texture_2d; // R8Unorm, sample .r @group(0) @binding(3) var samp: sampler; +// Masque de segmentation du sujet webcam, R8. Une vue 1x1 est liee quand aucun masque +// n'existe : la branche n'est de toute facon prise que si layer.fx.z > 0.5. +@group(0) @binding(4) var texMask: texture_2d; +// V est en binding 5 et pas 3 : les bindings 0-4 etaient deja pris quand le plan +// de chroma a ete dedouble, et renumeroter aurait touche tous les bind groups +// pour un gain nul. +@group(0) @binding(5) var texV: texture_2d; // R8Unorm, sample .r struct VsOut { @builtin(position) pos: vec4, @@ -69,7 +76,10 @@ fn yuv709_limited(y: f32, cbcr: vec2) -> vec3 { fn sample_yuv(uv: vec2) -> vec3 { let y = textureSample(texY, samp, uv).r; - let cbcr = textureSample(texUV, samp, uv).rg; + let cbcr = vec2( + textureSample(texU, samp, uv).r, + textureSample(texV, samp, uv).r, + ); return yuv709_limited(y, cbcr); } @@ -79,6 +89,20 @@ fn sd_round_rect(p: vec2, halfsz: vec2, r: f32) -> f32 { return length(max(q, vec2(0.0))) + min(max(q.x, q.y), 0.0) - r; } +// Couverture du quad avec coins arrondis, pour le mode 6 qui retourne AVANT la queue de +// `fs_main`. Il s'en passait tant qu'il ne servait qu'au fond plein cadre, qui n'a pas de +// rayon ; depuis que la bulle webcam peut porter une image, sans ca le fond deborde en carre +// opaque sur les coins arrondis de la bulle et mange l'ombre. Renvoie 1.0 quand aucun rayon +// n'est demande -- le fond plein cadre est donc inchange. +fn quad_round_alpha(local: vec2, quad_px: vec2, radius_px: f32) -> f32 { + if radius_px <= 0.0 || quad_px.x <= 0.0 || quad_px.y <= 0.0 { + return 1.0; + } + let halfsz = quad_px * 0.5; + let d = sd_round_rect(local - halfsz, halfsz, radius_px); + return 1.0 - smoothstep(0.0, 1.5, d); // meme feather ~1.5px que la queue +} + // ---- Primitives du tilt 3D (modes 8 et 12), portees de `shaders.metal` ---- // SDF segment a bouts ronds. @@ -197,10 +221,58 @@ fn quad_inverse_bilinear(P: vec2, c00: vec2, c10: vec2, c11: vec2 return r1; } +// Fond flouté pour le mode "blur" de la webcam. +// Disque de Vogel (spirale à angle d'or) à 21 échantillons avec pondération gaussienne et +// rotation par pixel via Interleaved Gradient Noise (IGN) pour un bokeh photographique doux, isotrope et rapide. +const VOGEL_TAPS = array, 21>( + vec3( 0.154303, 0.000000, 0.942213), + vec3(-0.197070, 0.180532, 0.836464), + vec3( 0.030165, -0.343712, 0.742584), + vec3( 0.248394, 0.323986, 0.659241), + vec3(-0.455834, -0.080631, 0.585251), + vec3( 0.431806, -0.274679, 0.519566), + vec3(-0.144431, 0.537274, 0.461253), + vec3(-0.275445, -0.530352, 0.409484), + vec3( 0.597605, 0.218244, 0.363526), + vec3(-0.621708, 0.256632, 0.322726), + vec3( 0.299704, -0.640451, 0.286505), + vec3( 0.221474, 0.706094, 0.254349), + vec3(-0.667525, -0.386844, 0.225802), + vec3( 0.783083, -0.172159, 0.200460), + vec3(-0.477903, 0.679768, 0.177961), + vec3(-0.110407, -0.852001, 0.157988), + vec3( 0.677789, 0.571241, 0.140256), + vec3(-0.912091, 0.037718, 0.124514), + vec3( 0.665301, -0.662063, 0.110540), + vec3(-0.044511, 0.962596, 0.098133), + vec3(-0.633036, -0.758588, 0.087119) +); + +fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2) -> vec3 { + let max_r_px = max(intensity, 0.0) * 22.0 + 1.5; + let step = max_r_px / max(qpx, vec2(1.0)); + let noise = fract(52.9829189 * fract(0.06711056 * local_px.x + 0.00583715 * local_px.y)); + let angle = noise * 6.2831853; + let s = sin(angle); + let c = cos(angle); + var sum = vec3(0.0); + var total = 0.0; + for (var k: i32 = 0; k < 21; k = k + 1) { + let p = VOGEL_TAPS[k].xy; + let w = VOGEL_TAPS[k].z; + let rot_p = vec2(p.x * c - p.y * s, p.x * s + p.y * c); + sum = sum + sample_yuv(clamp(uv + rot_p * step, vec2(0.0), vec2(1.0))) * w; + total = total + w; + } + return sum / max(total, 1e-4); +} + @fragment fn fs_main(i: VsOut) -> @location(0) vec4 { var rgb: vec3; var alpha: f32; + // 1 sauf en detourage, ou il porte le masque du sujet. Cf. la branche fx.z plus bas. + var alpha_mask = 1.0; if layer.mode < 0.5 { // Mode 0 — vidéo NV12 + flou de mouvement par vélocité (§8), port 1:1 du @@ -209,16 +281,18 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // floute le long de ce segment, ce qui capture la translation ET le zoom // du calque sans avoir à transporter un champ de vitesse. let taps = i32(layer.mb.x); + let mb_scale = clamp(layer.mb.y, 0.0, 1.0); // `taps` d'abord : un draw qui a oublié `dst_prev` le laisse à zéro, et // la division par `dst_prev.zw` produirait des UV infinis. Dégrader vers // le chemin net est le seul échec acceptable pour un effet cosmétique. - if taps <= 1 || layer.dst_prev.z <= 0.0 || layer.dst_prev.w <= 0.0 { + if taps <= 1 || mb_scale <= 0.001 || layer.dst_prev.z <= 0.0 || layer.dst_prev.w <= 0.0 { rgb = sample_yuv(i.uv); } else { let localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; let uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); let duv = i.uv - uv_prev; - if dot(duv, duv) < 1e-9 { + let duv_blur = duv * mb_scale; + if dot(duv_blur, duv_blur) < 1e-9 { rgb = sample_yuv(i.uv); } else { // Borne 16 en dur, identique au HLSL et au MSL : `taps` vient d'un @@ -229,18 +303,44 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let step = 1.0 / f32(taps - 1); for (var k: i32 = 0; k < 16; k = k + 1) { if k >= taps { break; } - acc = acc + sample_yuv(uv_prev + duv * (f32(k) * step)); + acc = acc + sample_yuv(i.uv - duv_blur * (1.0 - f32(k) * step)); } rgb = acc / f32(taps); } } + + // Effet d'arriere-plan webcam. Miroir exact des branches HLSL et MSL : fx.z porte le + // mode (1 = detourage, 2 = flou, 3 = fond plat), fx.w l'intensite du flou, fx.xy + // l'etendue valide de la texture webcam pour ramener uv dans l'espace du masque. + let effect = layer.fx.z; + if effect > 0.5 { + let mask_uv = i.uv / max(layer.fx.xy, vec2(1e-6)); + let person = clamp(textureSample(texMask, samp, mask_uv).r, 0.0, 1.0); + if effect > 2.5 { + rgb = mix(layer.color.rgb, rgb, person); + } else if effect > 1.5 { + rgb = mix(blur_webcam_bg(i.uv, layer.fx.w, layer.quad_px, i.local), rgb, person); + } else { + alpha_mask = person; + } + } } else if layer.mode < 1.5 { // Mode 1 — couleur pleine. rgb = layer.color.rgb; } else if layer.mode > 4.5 && layer.mode < 5.5 { // Mode 5 -- gradient lineaire : color (c0) -> src.rgb (c1) le long de // la direction fx.xy (sin, -cos de l'angle). Parite avec le HLSL/MSL. - let t = clamp(dot(i.pout - vec2(0.5), layer.fx.xy) + 0.5, 0.0, 1.0); + // `denom` : HLSL et MSL normalisent coin-a-coin (|dx|+|dy|) pour couvrir toute la + // diagonale. Il manquait ici, donc le meme degrade ne rendait pas pareil sur Linux. + let denom = max(abs(layer.fx.x) + abs(layer.fx.y), 1e-4); + // Parametre sur le QUAD des qu'il en a un (la bulle webcam), sinon sur la sortie. Pour + // le fond plein cadre les deux coincident ; pour une bulle dans un coin, `pout` ne + // montrerait que la tranche du degrade plein cadre qui passe dessous. + var gp = i.pout; + if layer.quad_px.x > 0.0 && layer.quad_px.y > 0.0 { + gp = i.local / layer.quad_px; + } + let t = clamp(0.5 + dot(gp - vec2(0.5), layer.fx.xy) / denom, 0.0, 1.0); rgb = mix(layer.color.rgb, layer.src.rgb, t); } else if layer.mode > 10.5 && layer.mode < 11.5 { // Mode 11 : texte. texY est l'atlas R8 (couverture alpha au canal .r, @@ -339,7 +439,8 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // Mode 6 -- fond image (wallpaper RGBA) cover-fit, echantillonne sur // texY. `src` porte le rect UV cover-fit (calcule cote Rust). Opaque : // le fond couvre tout le cadre. - return vec4(textureSample(texY, samp, i.uv).rgb, 1.0); + let bg_a = quad_round_alpha(i.local, layer.quad_px, layer.radius_px); + return vec4(textureSample(texY, samp, i.uv).rgb * bg_a, bg_a); // premultiplie } else if layer.mode > 7.5 && layer.mode < 8.5 { // Mode 8 -- ecran tilte (rotation 3D des zoom regions). Le quad projete est // dessine dans sa BBOX (le VS ne sait tracer qu'un rect) et chaque fragment @@ -427,7 +528,7 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { return vec4(layer.color.rgb * a, a); } - alpha = layer.color.a; + alpha = layer.color.a * alpha_mask; if layer.radius_px > 0.0 { // Feather ~1.5 px sur le bord du quad — parité exacte avec le HLSL diff --git a/crates/compositor/src/vk_shaders/yuv.wgsl b/crates/compositor/src/vk_shaders/yuv.wgsl new file mode 100644 index 000000000..68a1ed68a --- /dev/null +++ b/crates/compositor/src/vk_shaders/yuv.wgsl @@ -0,0 +1,83 @@ +// RGBA composee -> plans Y / U / V, sur le GPU. +// +// POURQUOI. Le chemin Linux relisait le RT en RGBA (8,3 Mo par frame en 1080p) +// puis convertissait en YUV420P sur le CPU avec `sws_scale`, mono-thread. +// Mesure sur un export S4 de 3600 frames : relecture 22,4 s, sws_scale 12,2 s. +// Convertir avant la relecture divise le volume relu par 2,67 (8,3 Mo -> 3,1 Mo) +// et fait disparaitre sws_scale. +// +// COEFFICIENTS. BT.601, plage limitee (Y 16-235, C 16-240) : c'est EXACTEMENT ce +// que `sws_getContext(..., SWS_POINT, ...)` produit par defaut pour une sortie +// YUV420P, et le but ici est d'accelerer la conversion, pas d'en changer le +// resultat. Toute autre matrice deplacerait les couleurs du fichier exporte. + +@group(0) @binding(0) var tex: texture_2d; +@group(0) @binding(1) var samp: sampler; + +struct VsOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, +}; + +// Meme triangle plein ecran que `blur.wgsl::vs_fullscreen`, meme orientation : +// une passe unique ne pardonne pas une erreur de sens (cf. la note la-bas). +@vertex +fn vs_fullscreen(@builtin(vertex_index) vid: u32) -> VsOut { + let pos = array, 3>( + vec2(-1.0, -1.0), + vec2( 3.0, -1.0), + vec2(-1.0, 3.0), + ); + let p = pos[vid]; + var o: VsOut; + o.pos = vec4(p, 0.0, 1.0); + o.uv = vec2(p.x * 0.5 + 0.5, 0.5 - p.y * 0.5); + return o; +} + +fn luma601(c: vec3) -> f32 { + return dot(c, vec3(0.299, 0.587, 0.114)); +} + +// Y pleine resolution. 16 + 219*Y', normalise sur 255 pour un R8Unorm. +@fragment +fn fs_y(i: VsOut) -> @location(0) vec4 { + let c = textureSample(tex, samp, i.uv).rgb; + let y = (16.0 + 219.0 * luma601(c)) / 255.0; + return vec4(y, 0.0, 0.0, 1.0); +} + +// U et V en demi-resolution. Le sampler est LINEAIRE et la cible fait la moitie +// de la source, donc echantillonner au centre du texel de sortie moyenne +// exactement le bloc 2x2 correspondant — le meme sous-echantillonnage que fait +// sws pour du 4:2:0, sans boucle de taps. +@fragment +fn fs_u(i: VsOut) -> @location(0) vec4 { + let c = textureSample(tex, samp, i.uv).rgb; + let u = (128.0 + 224.0 * (-0.168736 * c.r - 0.331264 * c.g + 0.5 * c.b)) / 255.0; + return vec4(u, 0.0, 0.0, 1.0); +} + +@fragment +fn fs_v(i: VsOut) -> @location(0) vec4 { + let c = textureSample(tex, samp, i.uv).rgb; + let v = (128.0 + 224.0 * (0.5 * c.r - 0.418688 * c.g - 0.081312 * c.b)) / 255.0; + return vec4(v, 0.0, 0.0, 1.0); +} + +// U ET V ENTRELACES, pour NV12. Meme mathematique et meme sous-echantillonnage +// que `fs_u`/`fs_v` : seule la destination change, un unique plan `Rg8Unorm` au +// lieu de deux plans `R8Unorm`. +// +// POURQUOI LES DEUX EXISTENT. `libopenh264` n'accepte que du YUV420P planaire +// (verifie : ses `pix_fmts` sont yuv420p/yuvj420p), tandis que VAAPI encode +// depuis du NV12. Le format n'est donc pas un gout mais une consequence de +// l'encodeur qui va consommer la frame, et le compositeur doit savoir produire +// les deux. +@fragment +fn fs_uv(i: VsOut) -> @location(0) vec4 { + let c = textureSample(tex, samp, i.uv).rgb; + let u = (128.0 + 224.0 * (-0.168736 * c.r - 0.331264 * c.g + 0.5 * c.b)) / 255.0; + let v = (128.0 + 224.0 * (0.5 * c.r - 0.418688 * c.g - 0.081312 * c.b)) / 255.0; + return vec4(u, v, 0.0, 1.0); +} diff --git a/crates/compositor/wrapper_linux.h b/crates/compositor/wrapper_linux.h index e01a31a97..70407440a 100644 --- a/crates/compositor/wrapper_linux.h +++ b/crates/compositor/wrapper_linux.h @@ -2,13 +2,27 @@ * * Software/VAAPI decode+encode only: no D3D11VA (Windows) nor VideoToolbox * (macOS) hwcontext headers, which pull platform-specific system headers - * (d3d11.h / CoreVideo) that don't exist on Linux. The generic hwcontext.h is - * kept for AVHWDeviceContext should the VAAPI path need it later. */ + * (d3d11.h / CoreVideo) that don't exist on Linux. + * + * hwcontext_drm.h IS included: the export path needs AVDRMFrameDescriptor to + * describe an exported dmabuf to av_hwframe_map, and that header pulls nothing + * beyond what the vendored ffmpeg already ships. + * + * hwcontext_vaapi.h is deliberately NOT included, though the export encodes with + * VAAPI. It #includes , which is not in the vendored tree -- adding it + * would make libva's DEVELOPMENT headers a build dependency of this crate on + * every Linux builder, to gain nothing: reaching VAAPI needs only + * AV_HWDEVICE_TYPE_VAAPI, an enumerator of the generic hwcontext.h, and + * av_hwdevice_ctx_create_derived. AVVAAPIDeviceContext itself is never touched. */ #include #include #include +#include #include #include #include #include #include +#include +#include +#include diff --git a/crates/compositor/wrapper_macos.h b/crates/compositor/wrapper_macos.h index 5a87a1466..f3a5e5dd9 100644 --- a/crates/compositor/wrapper_macos.h +++ b/crates/compositor/wrapper_macos.h @@ -17,4 +17,7 @@ /* Software decode path : swscale était déjà LIÉ (build.rs) sans être bindé. Conservé identique côté macOS pour que la symétrie avec cpu_frames_windows.rs soit claire ; le code effectif vit dans mac_frames.rs. */ -#include \ No newline at end of file +#include +#include +#include +#include diff --git a/crates/compositor/wrapper_windows.h b/crates/compositor/wrapper_windows.h index 86612f96a..8d291a813 100644 --- a/crates/compositor/wrapper_windows.h +++ b/crates/compositor/wrapper_windows.h @@ -12,3 +12,6 @@ elle couvre les formats exotiques (10 bits, 4:2:2) qu'un interleave écrit à la main casserait silencieusement. */ #include +#include +#include +#include diff --git a/crates/poc-d3d/src/bench.rs b/crates/poc-d3d/src/bench.rs index aa2de7d60..34460d9aa 100644 --- a/crates/poc-d3d/src/bench.rs +++ b/crates/poc-d3d/src/bench.rs @@ -150,6 +150,7 @@ fn run_bench(args: &[String]) -> Result<()> { source_end_sec: 6.0, // la fixture entière (§ fixture.json : 6 s, 360 frames) webcam_offset_sec: 0.0, has_audio: false, + hold_sec: 0.0, }; let path = format!("{out}/{}_{:?}.mp4", cfg.name, backend).to_lowercase(); let s = pipeline::run_composited_multi( @@ -292,6 +293,7 @@ fn run_gif_bench( source_end_sec: f64::MAX, webcam_offset_sec: 0.0, has_audio: false, + hold_sec: 0.0, }]; for r in 0..repeat { // Each run writes to the same path — the last frame wins. The diff --git a/electron-builder.json5 b/electron-builder.json5 index 998d38ae3..dafe8c690 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -10,6 +10,17 @@ "**/*.node" ], "productName": "Openscreen", + // There is deliberately no `electronVersion` here. electron-builder needs an EXACT + // version because it downloads the binaries for one release, and it takes that from + // package.json's `electron` devDependency, which is pinned without a caret for this + // reason. Setting it here as well would make the version two facts that have to agree, + // and nothing would notice when they stopped. + // + // The error that leads people here is "Electron version ... is a range, not a fixed + // version", raised when the project has no `node_modules/electron` of its own — a git + // worktree, typically, where module resolution finds the parent checkout's copy but + // electron-builder looks only at the project's. The fix is `npm ci` in the worktree, not + // a second declaration of the version. // Declared rather than left to default BECAUSE the default is package.json's `author` — one // name, where LICENSE has two holders. Feeds Info.plist's NSHumanReadableCopyright and the // Windows LegalCopyright, so leaving it implicit put an attribution on the binary that the @@ -97,10 +108,30 @@ { "from": "public/cursors", "to": "cursors" + }, + // The segmentation model, out of the asar for the same reason as the two above: the main + // process resolves it with `original-fs` (see realExistsSync in compositorViewService.ts), + // which cannot see inside app.asar, so a copy that ships only in `dist` resolves to null and + // the compositor draws the webcam unsegmented — the whole camera-background feature, silently + // off in every installer. 457 KB. + { + "from": "public/mediapipe", + "to": "mediapipe" } ], "mac": { + // Declared, not merely documented. Electron 41's own LSMinimumSystemVersion is 12.0 + // and the .app inherits it verbatim when this key is absent — so before this line the + // bundle advertised macOS 12 while its native payload was built for 13, and a + // Monterey user got as far as the record button before anything went wrong (#515). + // LaunchServices now refuses to open the app below 13 instead, which is the honest + // signal. + // + // macOS 13 because ScreenCaptureKit capture requires it: ScreenCaptureRecorder is + // `@available(macOS 13.0, *)`. Keep in step with README.md, website/docs/ + // installation.md, and electron/native/screencapturekit/Package.swift. + "minimumSystemVersion": "13.0", "notarize": false, "hardenedRuntime": true, "entitlements": "macos.entitlements", diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 8f8361e59..20d345298 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -172,6 +172,7 @@ describe("the mutating-tool table", () => { expect([...MUTATING_TOOL_NAMES].sort()).toEqual( [ "addAnnotation", + "addAudio", "addCameraFullscreen", "addSpeed", "addTrim", @@ -184,10 +185,12 @@ describe("the mutating-tool table", () => { "removeTrim", "replaceTimeline", "setAnnotation", + "setAudio", "setCameraFullscreen", "setClipRange", "setSpeed", "setTrim", + "setWordText", "setZoom", ].sort(), ); @@ -2054,3 +2057,324 @@ describe("setZoom answers for the focus it kept", () => { expect(result.resultJson).not.toContain("cursorAnchor"); }); }); + +// Issue #350 / #560 — the audio tools. #561 landed the timeline audio without any +// agent surface, so these cover both that the model can see it and that it cannot +// invent an asset to place. +describe("addAudio / setAudio", () => { + /** The fixture plus one imported audio asset. */ + function withAudioAsset(durationSec: number | null = 30): AxcutDocument { + const doc = fixtureDocument(); + return documentSchema.parse({ + ...doc, + assets: [ + ...doc.assets, + { + id: "audio_1", + kind: "audio", + label: "bed.mp3", + originalPath: "C:/audio/bed.mp3", + ...(durationSec == null ? {} : { durationSec }), + }, + ], + }); + } + + const place = (doc: AxcutDocument, args: Record) => + executeAgentTool(doc, "addAudio", JSON.stringify(args)); + + it("reports imported audio in the snapshot, with the asset kind beside it", () => { + // Without `kind` the model sees an asset it cannot explain and tries to place it + // as footage; without `audioTracks` it cannot see the lanes at all. + const placed = place(withAudioAsset(), { + assetId: "audio_1", + startSec: 2, + endSec: 6, + kind: "voiceover", + }); + expect(placed.ok).toBe(true); + const snapshot = executeAgentTool(placed.document as AxcutDocument, "getCurrentDocument", ""); + const parsed = JSON.parse(snapshot.resultJson); + expect(parsed.assets.find((a: { id: string }) => a.id === "audio_1").kind).toBe("audio"); + expect(parsed.audioTracks).toHaveLength(1); + expect(parsed.audioTracks[0]).toMatchObject({ + assetId: "audio_1", + kind: "voiceover", + startSec: 2, + endSec: 6, + }); + }); + + it("anchors the placed track to the clip under it", () => { + const result = place(withAudioAsset(), { assetId: "audio_1", startSec: 2, endSec: 6 }); + expect(result.ok).toBe(true); + const track = (result.document as AxcutDocument).audioTracks[0]; + // The anchor is what makes it travel with its clip; a bare startMs/endMs would not. + expect(track.clipId).toBe("clip_1"); + expect(track.origin).toBe("agent"); + }); + + it("plays the whole file when endSec is omitted", () => { + const result = place(withAudioAsset(20), { assetId: "audio_1", startSec: 0, offsetSec: 5 }); + expect(result.ok).toBe(true); + const track = (result.document as AxcutDocument).audioTracks[0]; + // 20s file from an in-point of 5s = 15s of span, so the model never computes it. + expect(track.endMs - track.startMs).toBe(15_000); + }); + + it("refuses an offset at or past the end of a known file", () => { + // Otherwise the omitted-end fallback mints a 0.1s track that plays silence, and + // the model reports it as having placed audio. + const result = place(withAudioAsset(20), { assetId: "audio_1", startSec: 0, offsetSec: 20 }); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("offsetSec"); + }); + + it("allows any offset while the duration is unknown", () => { + // A failed probe leaves no duration; refusing on that would block a legitimate call. + expect(place(withAudioAsset(null), { assetId: "audio_1", startSec: 0, offsetSec: 99 }).ok).toBe( + true, + ); + }); + + it("refuses an unknown asset and names the audio the project actually has", () => { + const result = place(withAudioAsset(), { assetId: "nope", startSec: 0, endSec: 4 }); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("audio_1"); + }); + + it("refuses a video asset, pointing at the tool that does place footage", () => { + const result = place(withAudioAsset(), { assetId: "asset_1", startSec: 0, endSec: 4 }); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("replaceTimeline"); + }); + + it("setAudio re-levels and re-lanes the track it names", () => { + const placed = place(withAudioAsset(), { assetId: "audio_1", startSec: 2, endSec: 6 }); + const id = JSON.parse(placed.resultJson).audioId; + const result = executeAgentTool( + placed.document as AxcutDocument, + "setAudio", + JSON.stringify({ audioId: id, gainDb: -6, kind: "voiceover" }), + ); + expect(result.ok).toBe(true); + expect((result.document as AxcutDocument).audioTracks[0]).toMatchObject({ + gainDb: -6, + kind: "voiceover", + }); + }); + + it("setAudio applies the same offset guard as addAudio", () => { + const placed = place(withAudioAsset(20), { assetId: "audio_1", startSec: 2, endSec: 6 }); + const id = JSON.parse(placed.resultJson).audioId; + const result = executeAgentTool( + placed.document as AxcutDocument, + "setAudio", + JSON.stringify({ audioId: id, offsetSec: 25 }), + ); + expect(result.ok).toBe(false); + }); + + it("removeModifier deletes an audio track by id, like every other kind", () => { + const placed = place(withAudioAsset(), { assetId: "audio_1", startSec: 2, endSec: 6 }); + const id = JSON.parse(placed.resultJson).audioId; + const result = executeAgentTool( + placed.document as AxcutDocument, + "removeModifier", + JSON.stringify({ id }), + ); + expect(result.ok).toBe(true); + expect((result.document as AxcutDocument).audioTracks).toEqual([]); + }); +}); + +// ─── Correcting a word from the chat ───────────────────────────── +// The model could READ the transcript and CUT it, and that was all. Asked to fix a +// misheard name it had exactly one tool that touched a word — addTrim — which removes the +// audio with it. These two close that: one read that hands out word ids, one write that +// changes text and nothing else. + +/** A transcript with real words, one of them already corrected by the user. */ +function documentWithWords(): AxcutDocument { + const base = fixtureDocument(); + return { + ...base, + transcripts: [ + { + assetId: "asset_1", + language: "en", + segments: [ + { + id: "seg_1", + kind: "speech", + startSec: 0, + endSec: 3, + text: "I use Cuber Nettes", + wordIds: ["word_1", "word_2", "word_3"], + }, + ], + words: [ + { id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 1, text: "I" }, + { id: "word_2", segmentId: "seg_1", startSec: 1, endSec: 2, text: "use" }, + { + id: "word_3", + segmentId: "seg_1", + startSec: 2, + endSec: 3, + text: "Cuber Nettes", + }, + ], + }, + ], + }; +} + +function run(document: AxcutDocument, name: string, args: unknown) { + return executeAgentTool(document, name, JSON.stringify(args), { editsAllowed: true }); +} + +describe("getTranscriptWords", () => { + it("hands out the ids setWordText takes", () => { + const result = run(documentWithWords(), "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string; text: string }>; + total: number; + }; + expect(result.ok).toBe(true); + expect(payload.total).toBe(3); + expect(payload.words.map((w) => w.id)).toEqual(["word_1", "word_2", "word_3"]); + }); + + // A half-hour transcript is ~70k tokens. Fixing one name should cost one phrase. + it("returns only the words touching the span it is given", () => { + const result = run(documentWithWords(), "getTranscriptWords", { startSec: 2, endSec: 3 }); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string }>; + total: number; + }; + // Touching counts: `word_2` ends exactly where the span begins. Inclusive on + // purpose — a word with no duration at all (one the user typed in) sits on a + // single point, and a strict overlap would drop it from every span it meets. + expect(payload.words.map((w) => w.id)).toEqual(["word_2", "word_3"]); + // `total` still reports the whole transcript, so a filtered read never reads as + // the entire thing. + expect(payload.total).toBe(3); + }); + + it("says nothing about provenance for a plainly transcribed word", () => { + const result = run(documentWithWords(), "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { words: Array> }; + expect(payload.words[0]).not.toHaveProperty("source"); + expect(payload.words[0]).not.toHaveProperty("originalText"); + }); + + it("names what the transcriber had heard, once a word is corrected", () => { + const corrected = run(documentWithWords(), "setWordText", { + wordId: "word_3", + text: "Kubernetes", + }); + const result = run(corrected.document as AxcutDocument, "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string; source?: string; originalText?: string }>; + }; + expect(payload.words.find((w) => w.id === "word_3")).toMatchObject({ + source: "user", + originalText: "Cuber Nettes", + }); + }); + + it("refuses an asset with no transcript instead of answering with nothing", () => { + const result = run({ ...fixtureDocument(), transcripts: [] }, "getTranscriptWords", {}); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("No transcript"); + }); +}); + +describe("setWordText", () => { + it("changes the text and leaves the timeline alone", () => { + const before = documentWithWords(); + const result = run(before, "setWordText", { wordId: "word_3", text: "Kubernetes" }); + expect(result.ok).toBe(true); + const next = result.document as AxcutDocument; + expect(next.transcripts[0].words.find((w) => w.id === "word_3")?.text).toBe("Kubernetes"); + expect(next.timeline).toEqual(before.timeline); + expect(next.transcripts[0].segments[0].text).toBe("I use Kubernetes"); + }); + + // The editor gates word INSERTION on a dev-only flag, and that gate lives in the renderer. + // The chat runs in the main process, so an ungated path here would let a release rewrite + // generated media through the agent — the one door the flag cannot see. + it("refuses a word that was added rather than heard", () => { + const base = documentWithWords(); + const withInsertion: AxcutDocument = { + ...base, + transcripts: [ + ...base.transcripts, + { + assetId: "ext:synth_1", + language: "en", + segments: [], + words: [ + { + id: "synth_1", + segmentId: "seg_1", + startSec: 0, + endSec: 0.15, + text: "added", + source: "synth", + }, + ], + }, + ], + }; + const result = run(withInsertion, "setWordText", { + assetId: "ext:synth_1", + wordId: "synth_1", + text: "much longer", + }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + }); + + // The document carries the transcript twice; a write that reaches only one leaves the + // legacy mirror serving the old text forever. + it("writes the legacy mirror too", () => { + const result = run(documentWithWords(), "setWordText", { + wordId: "word_3", + text: "Kubernetes", + }); + const next = result.document as AxcutDocument; + expect(next.transcript).toBe(next.transcripts.find((t) => t.assetId === "asset_1")); + }); + + it("empties a word without cutting the speech around it", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "" }); + const next = result.document as AxcutDocument; + expect(next.transcripts[0].words.find((w) => w.id === "word_2")?.text).toBe(""); + expect(next.transcripts[0].segments[0].text).toBe("I Cuber Nettes"); + expect(JSON.parse(result.resultJson)).toMatchObject({ blanked: true }); + }); + + it("points an unknown id at the read that hands them out", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "seg_1", text: "x" }); + expect(result.ok).toBe(false); + // `seg_1` is a real id — of a SEGMENT. The two namespaces are the trap. + expect(result.resultJson).toContain("getTranscriptWords"); + }); + + it("refuses a write that would change nothing", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "use" }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + }); + + it("is a consented edit, not a read", () => { + const result = executeAgentTool( + documentWithWords(), + "setWordText", + JSON.stringify({ wordId: "word_3", text: "Kubernetes" }), + { editsAllowed: false }, + ); + expect(result.document).toBeUndefined(); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 5a0966398..cfccf9c06 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -16,6 +16,12 @@ // described are gone (see `MUTATING_TOOL_NAMES`). import { z } from "zod"; +import { + collapseTracksToPills, + patchAudioTrack, + placeAudioTrackInDocument, + trackGroupId, +} from "../../src/lib/ai-edition/document/audioTracks"; import { createId } from "../../src/lib/ai-edition/document/ids"; import { moveClip, @@ -26,8 +32,10 @@ import { replaceTimeline, setClipSourceRange, } from "../../src/lib/ai-edition/document/timeline"; +import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript"; import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; +import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts"; import { buildCursorTrack, type CursorTrackSample, @@ -337,6 +345,11 @@ function droppedByEdit(before: AxcutDocument, after: AxcutDocument) { // private — callers only ever need the composed `*Args`.) const secondsSchema = z.number().finite().nonnegative(); +/** Span given to an agent-placed audio track when the asset has no probed duration + * yet. Short on purpose: a wrong guess the user has to lengthen beats one that + * silently covers the whole programme. */ +const DEFAULT_AGENT_AUDIO_SEC = 10; + export const addTrimArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, @@ -479,6 +492,26 @@ export const setAnnotationArgs = z.object({ text: z.string().optional(), }); +export const addAudioArgs = z.object({ + assetId: z.string().min(1), + startSec: secondsSchema, + endSec: secondsSchema.optional(), + kind: z.enum(["voiceover", "music"]).default("music"), + offsetSec: secondsSchema.default(0), + gainDb: z.number().min(-60).max(12).default(0), +}); + +export const setAudioArgs = z.object({ + audioId: z.string().min(1), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), + kind: z.enum(["voiceover", "music"]).optional(), + offsetSec: secondsSchema.optional(), + gainDb: z.number().min(-60).max(12).optional(), + muted: z.boolean().optional(), + loop: z.boolean().optional(), +}); + export const addCameraFullscreenArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, @@ -490,6 +523,18 @@ export const setCameraFullscreenArgs = z.object({ endSec: secondsSchema.optional(), }); +export const getTranscriptWordsArgs = z.object({ + assetId: z.string().min(1).optional(), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), +}); + +export const setWordTextArgs = z.object({ + wordId: z.string().min(1), + text: z.string(), + assetId: z.string().min(1).optional(), +}); + export const removeTrimArgs = z.object({ trimRangeId: z.string().min(1), }); @@ -525,7 +570,9 @@ export const removeClipArgs = z.object({ export const OPENSCREEN_TOOL_NAMES = [ "getCurrentDocument", "getTranscript", + "getTranscriptWords", "getCursorTrack", + "setWordText", "addTrim", "addTrims", "setTrim", @@ -541,6 +588,8 @@ export const OPENSCREEN_TOOL_NAMES = [ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addAudio", + "setAudio", "removeTrim", "removeModifier", "removeClip", @@ -592,6 +641,9 @@ export const PHANTOM_TOOL_NAMES = [ * remaining surfaces (descriptions, built tools, executor cases) to each other. */ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ + // Writes the transcript, not the timeline — but it writes the document, so it is a + // consented edit like any other. + "setWordText", "addTrim", "addTrims", "addZooms", @@ -607,6 +659,8 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addAudio", + "setAudio", "removeTrim", "removeModifier", "removeClip", @@ -665,7 +719,9 @@ export function documentSnapshotForModel( const autoFocusAll = legacy?.autoFocusAll === true; return { timeBaseNote: - "clips and trims are in source-time seconds; zooms, speedRegions, annotations and cameraFullscreenRegions are in virtual (edited-timeline) seconds.", + "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds.", + audioNote: + "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.", zoomNote: `renderedScale is what the viewer sees (depth is an ordinal, not a factor: ${ZOOM_DEPTH_LEGEND}). ` + "When a zoom carries customScale it wins over depth and depthIsOverridden is true — " + @@ -683,6 +739,10 @@ export function documentSnapshotForModel( assets: document.assets.map((a) => ({ id: a.id, label: a.label, + // "audio" is an imported voiceover / music file: it is never a clip, it is + // played by an audio track. Without this the model sees an asset it cannot + // explain and tries to place it on the timeline as footage. + kind: a.kind, durationSec: a.durationSec ?? null, hasCameraTrack: a.cameraTrack != null, cameraVisible: a.cameraTrack?.visible ?? false, @@ -755,6 +815,22 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + // Imported audio, collapsed to the pills the ruler draws — a track ventilated + // across a clip boundary is several fragments the user sees as one thing, and + // the model has to name what the user sees. + audioTracks: collapseTracksToPills(document.audioTracks).map((t) => ({ + id: trackGroupId(t), + startSec: roundSec(t.startMs), + endSec: roundSec(t.endMs), + assetId: t.assetId, + // Which lane it sits on. Also decides whether it is transcribed at all. + kind: t.kind, + // Where in the FILE the track starts playing, in that file's own seconds. + offsetSec: roundSec(t.offsetMs), + gainDb: t.gainDb, + muted: t.muted, + loop: t.loop, + })), hasTranscript: document.transcripts.length > 0 || document.transcript !== null, }; } @@ -1203,6 +1279,107 @@ export function executeAgentTool( }; } + // The word-level read. `getTranscript` answers in SEGMENTS, whose ids belong to a + // different namespace than the words — so on its own it cannot address anything + // `setWordText` takes. This is the one that can. It is separate rather than folded + // in because a whole transcript is already ~70k tokens and most turns never touch a + // word; the span filter is there so fixing one name costs one phrase, not the film. + case "getTranscriptWords": { + const parsed = getTranscriptWordsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const assetId = + parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id; + const transcript = + document.transcripts.find((t) => t.assetId === assetId) ?? + (document.transcript?.assetId === assetId ? document.transcript : null); + if (!transcript) { + return failure(`No transcript for asset ${assetId ?? "(none)"}.`); + } + const from = parsed.data.startSec ?? Number.NEGATIVE_INFINITY; + const to = parsed.data.endSec ?? Number.POSITIVE_INFINITY; + const words = transcript.words + .filter((word) => word.endSec >= from && word.startSec <= to) + .map((word) => ({ + id: word.id, + text: word.text, + startSec: word.startSec, + endSec: word.endSec, + // Only the words that are NOT plain transcription say so, so the common + // case costs nothing to read. + ...(word.source ? { source: word.source } : {}), + ...(word.originalText !== undefined ? { originalText: word.originalText } : {}), + })); + return { + ok: true, + resultJson: JSON.stringify({ + assetId, + language: transcript.language, + total: transcript.words.length, + returned: words.length, + words, + }), + }; + } + + // Correcting what the transcriber HEARD. This writes text and nothing else: the + // captions follow it, the film does not move. The tool for making a spoken word go + // away is addTrim, which removes its audio with it. + case "setWordText": { + const parsed = setWordTextArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const assetId = + parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id; + if (!assetId) return failure("Project has no assets — nothing to correct."); + const { wordId, text } = parsed.data; + const transcript = document.transcripts.find((t) => t.assetId === assetId); + const before = transcript?.words.find((word) => word.id === wordId); + if (!before) { + return failure( + `No word ${wordId} in the transcript for asset ${assetId}. ` + + `Call getTranscriptWords to read the ids.`, + ); + } + if (before.text === text) { + return failure(`Word ${wordId} already reads "${text}" — nothing to change.`); + } + // This tool exists to fix a name the transcriber misheard. An INSERTED word was + // never heard: retyping it resizes the clip it plays on and asks for generated + // media of a new length, which is the gesture the editor gates on `insertionsEnabled` + // — and that gate lives in the renderer, where the chat does not run. Refused here + // unconditionally rather than mirrored, because the agent has no business authoring + // generated media at all. + if (isGeneratedAssetId(assetId)) { + return failure( + `Word ${wordId} was added to the transcript, not heard — the chat cannot rewrite it.`, + ); + } + let next: AxcutDocument; + try { + next = setDocumentWordText(document, assetId, wordId, text); + } catch (error) { + return failure(error instanceof Error ? error.message : String(error)); + } + const after = next.transcripts + .find((t) => t.assetId === assetId) + ?.words.find((word) => word.id === wordId); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + wordId, + assetId, + text: after?.text ?? text, + was: before.text, + // Absent once the word is back to what the transcriber said — the pair is + // cleared on that round trip, and the model should be able to see it. + originalText: after?.originalText, + blanked: text.trim().length === 0, + }), + summary: + text.trim().length === 0 ? `blanked "${before.text}"` : `"${before.text}" → "${text}"`, + }; + } + case "addTrim": { const parsed = addTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -1843,6 +2020,165 @@ export function executeAgentTool( }; } + case "addAudio": { + const parsed = addAudioArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { assetId, kind, offsetSec, gainDb } = parsed.data; + const asset = document.assets.find((a) => a.id === assetId); + // Two distinct refusals, because they need two different corrections: an + // unknown id is a hallucinated asset, a video id is the model reaching for + // footage. Naming the audio the project HAS is what stops the retry loop. + if (!asset) { + const available = document.assets.filter((a) => a.kind === "audio"); + return failure( + `Unknown asset: ${assetId}.` + + (available.length + ? ` Imported audio in this project: ${available.map((a) => `${a.id} (${a.label})`).join(", ")}.` + : " This project has no imported audio; a file can only be imported or recorded from the editor, not from here."), + ); + } + if (asset.kind !== "audio") { + return failure( + `Asset ${assetId} is video, not audio. addAudio plays an imported audio file over the recording; to place footage use replaceTimeline.`, + ); + } + const durationSec = asset.durationSec ?? 0; + // "Start the file at offsetSec" is only answerable when there is file left + // there. Past the end it yields a track that plays silence, which the model + // then reports as having placed audio. Unknown duration is not a refusal: an + // import whose probe failed carries 0 until the renderer re-probes it. + if (durationSec > 0 && offsetSec >= durationSec) { + return failure( + `offsetSec ${offsetSec}s is at or past the end of ${assetId} (${durationSec}s), so the track would play nothing. Pick an offset inside the file.`, + ); + } + // No endSec means "as long as the file is" — the natural span, and the one + // the editor's own add uses, so the model never has to compute it. + const startSec = parsed.data.startSec; + const endSec = + parsed.data.endSec ?? + startSec + Math.max(0.1, (durationSec || DEFAULT_AGENT_AUDIO_SEC) - offsetSec); + const startMs = toMs(Math.min(startSec, endSec)); + const endMs = toMs(Math.max(startSec, endSec)); + const trackId = createId("audio"); + const withTrack = placeAudioTrackInDocument( + document, + { + id: trackId, + trackId, + startMs, + endMs, + assetId, + kind, + durationSec, + offsetMs: toMs(offsetSec), + gainDb, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: asset.label, + origin: "agent", + } as AxcutDocument["audioTracks"][number], + () => createId("audio"), + "create", + ); + if (withTrack === document) { + return coversNoClip("audio", startMs / 1000, endMs / 1000, document); + } + const placed = withTrack.audioTracks.filter((t) => trackGroupId(t) === trackId); + const next: AxcutDocument = withTrack; + const landing = landingOf(placed, document); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + audioId: trackId, + ...landingReport(landing, startMs / 1000, endMs / 1000), + }), + summary: + `added ${kind} "${asset.label}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + + case "setAudio": { + const parsed = setAudioArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { audioId } = parsed.data; + const pill = collapseTracksToPills(document.audioTracks).find( + (t) => trackGroupId(t) === audioId, + ); + if (!pill) return failure(`Unknown audio track: ${audioId}`); + + if (parsed.data.offsetSec !== undefined) { + const asset = document.assets.find((a) => a.id === pill.assetId); + const durationSec = asset?.durationSec ?? 0; + if (durationSec > 0 && parsed.data.offsetSec >= durationSec) { + return failure( + `offsetSec ${parsed.data.offsetSec}s is at or past the end of ${pill.assetId} (${durationSec}s), so the track would play nothing.`, + ); + } + } + + // Payload first, through the helper that keeps every fragment of the group in + // agreement — gain, mute, loop and the offset are all track-wide, and a patch + // that reached only one fragment would split the pill in two. + let next = patchAudioTrack(document, audioId, { + ...(parsed.data.gainDb !== undefined ? { gainDb: parsed.data.gainDb } : {}), + ...(parsed.data.muted !== undefined ? { muted: parsed.data.muted } : {}), + ...(parsed.data.loop !== undefined ? { loop: parsed.data.loop } : {}), + ...(parsed.data.offsetSec !== undefined ? { offsetMs: toMs(parsed.data.offsetSec) } : {}), + }); + + // A span or lane change re-anchors: drop the group and lay it down again, so + // the fragments are re-cut against the clips the new span covers rather than + // patched in place against the old ones. + const wantsRespan = + parsed.data.startSec !== undefined || + parsed.data.endSec !== undefined || + parsed.data.kind !== undefined; + if (wantsRespan) { + const current = + collapseTracksToPills(next.audioTracks).find((t) => trackGroupId(t) === audioId) ?? pill; + const { startMs, endMs } = resolveSpanMs(current, parsed.data.startSec, parsed.data.endSec); + // A `kind` flip re-clamps against the DESTINATION lane's neighbours, not the + // one it is leaving — moving a take onto the music row must respect what is + // already on the music row (issue #560). + const moved = placeAudioTrackInDocument( + next, + { + ...current, + id: audioId, + trackId: audioId, + startMs, + endMs, + ...(parsed.data.kind !== undefined ? { kind: parsed.data.kind } : {}), + }, + () => createId("audio"), + "move", + ); + if (moved === next) { + return coversNoClip("audio", startMs / 1000, endMs / 1000, document); + } + next = moved; + } + + const after = collapseTracksToPills(next.audioTracks).find( + (t) => trackGroupId(t) === audioId, + ); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + audioId, + startSec: roundSec(after?.startMs ?? pill.startMs), + endSec: roundSec(after?.endMs ?? pill.endMs), + }), + summary: `updated audio ${audioId} ${formatSec(roundSec(after?.startMs ?? pill.startMs))} – ${formatSec(roundSec(after?.endMs ?? pill.endMs))}`, + }; + } + case "removeTrim": { const parsed = removeTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -1872,9 +2208,10 @@ export function executeAgentTool( else if (document.annotations.some((a) => a.id === id)) kind = "annotation"; else if (speedRegions.some((s) => s.id === id)) kind = "speed"; else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen"; + else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio"; if (!kind) { return failure( - `No zoom / speed / annotation / full-camera modifier with id ${id}. ` + + `No zoom / speed / annotation / full-camera / audio modifier with id ${id}. ` + `For a trim use removeTrim; for a clip use removeClip.`, ); } diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index 4dcac3423..20e635062 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -438,7 +438,7 @@ export async function runChat( success: true, assistantMessage, // ponytail: belt and braces on `editsAllowed`. This returned document is - // the ONLY path to disk (LeftPanel → applyAgentDocument → saveDocument), + // the ONLY path to disk (ChatStripPanel → applyAgentDocument → saveDocument), // so it is where a write that somehow escaped the executor's guard would // still land. Cheap, and it makes the setting's guarantee structural // rather than dependent on one predicate holding everywhere. diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 7729bd624..5aebb9a33 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -57,7 +57,9 @@ const PHANTOM_TOOLS: readonly string[] = PHANTOM_TOOL_NAMES; const ARGS: Record = { getCurrentDocument: {}, getTranscript: {}, + getTranscriptWords: {}, getCursorTrack: {}, + setWordText: { wordId: "word_1", text: "Hullo" }, addTrim: { startSec: 1, endSec: 2 }, addTrims: { ranges: [{ startSec: 1, endSec: 2 }] }, setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 }, @@ -73,6 +75,10 @@ const ARGS: Record = { setAnnotation: { annotationId: "ann_nope" }, addCameraFullscreen: { startSec: 1, endSec: 2 }, setCameraFullscreen: { cameraFullscreenId: "cam_nope" }, + // The fixture has no `kind: "audio"` asset, so these exercise the refusal branch — + // the honest one to pin: the agent can place imported audio, never import it. + addAudio: { assetId: "audio_nope", startSec: 1, endSec: 2 }, + setAudio: { audioId: "audio_nope" }, removeTrim: { trimRangeId: "trim_1" }, removeModifier: { id: "nope" }, removeClip: { clipId: "clip_1" }, @@ -109,9 +115,18 @@ function fixtureDocument(): AxcutDocument { assetId: "asset_1", language: "en", segments: [ - { id: "seg_1", kind: "speech", startSec: 0, endSec: 5, text: "Hello", wordIds: [] }, + { + id: "seg_1", + kind: "speech", + startSec: 0, + endSec: 5, + text: "Hello", + // A real word, so `setWordText` lands on its WRITE branch in the table + // below — a tool refused for an unknown id would look non-mutating. + wordIds: ["word_1"], + }, ], - words: [], + words: [{ id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 5, text: "Hello" }], }, ], timeline: { @@ -351,6 +366,7 @@ describe("one description of the tools, not two", () => { expect(OPENSCREEN_TOOLS.filter((n) => !isMutatingTool(n))).toEqual([ "getCurrentDocument", "getTranscript", + "getTranscriptWords", "getCursorTrack", ]); }); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index d804a3b1a..7ddc55bfc 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -27,6 +27,7 @@ import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { addAnnotationArgs, + addAudioArgs, addCameraFullscreenArgs, addSpeedArgs, addTrimArgs, @@ -37,6 +38,7 @@ import { executeAgentTool, getCursorTrackArgs, getTranscriptArgs, + getTranscriptWordsArgs, isMutatingTool, moveClipArgs, removeClipArgs, @@ -45,10 +47,12 @@ import { replaceTimelineArgs, resolveCursorAssetId, setAnnotationArgs, + setAudioArgs, setCameraFullscreenArgs, setClipRangeArgs, setSpeedArgs, setTrimArgs, + setWordTextArgs, setZoomArgs, } from "../agent-tools"; import { @@ -115,6 +119,7 @@ const BASE_SYSTEM_PROMPT = [ "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, + "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", "- Deleting is a first-class action, not a workaround: removeTrim, removeModifier, removeClip. Never fake a deletion by re-adding an element or zeroing it out (span 0, speed 1×) — that leaves it in the document and misreports what you did.", "If nothing in the list does what was asked, say so; do not approximate it with a bigger tool.", @@ -143,6 +148,10 @@ export const TOOL_DESCRIPTIONS: Record = { "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", + getTranscriptWords: + 'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.', + setWordText: + "Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use addTrim, which cuts the audio with it.", addTrim: "Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).", addTrims: @@ -170,10 +179,14 @@ export const TOOL_DESCRIPTIONS: Record = { "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", + addAudio: + "Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.", + setAudio: + "Move, resize, re-level, re-lane, mute, loop or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.", removeTrim: "Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.", removeModifier: - "Delete a modifier (zoom / speed / annotation / camera-fullscreen) by id; the kind is resolved from the id. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", + "Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio) by id; the kind is resolved from the id. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", removeClip: "Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.", }; @@ -323,7 +336,9 @@ export function buildTools( return [ build("getCurrentDocument", z.object({})), build("getTranscript", getTranscriptArgs), + build("getTranscriptWords", getTranscriptWordsArgs), build("getCursorTrack", getCursorTrackArgs), + build("setWordText", setWordTextArgs), build("addTrim", addTrimArgs), build("addTrims", addTrimsArgs), build("setTrim", setTrimArgs), @@ -339,6 +354,8 @@ export function buildTools( build("setAnnotation", setAnnotationArgs), build("addCameraFullscreen", addCameraFullscreenArgs), build("setCameraFullscreen", setCameraFullscreenArgs), + build("addAudio", addAudioArgs), + build("setAudio", setAudioArgs), build("removeTrim", removeTrimArgs), build("removeModifier", removeModifierArgs), build("removeClip", removeClipArgs), diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 6cbdb97c9..5951c9090 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -245,6 +245,34 @@ describe("DocumentService", () => { }); }); + describe("onProjectRead", () => { + it("announces every document it hands out, after the relink", async () => { + // The read allow-list lives in the main process and is in memory: a picker's + // approval is gone by the next launch. This callback is how a project reopened + // tomorrow can still read the media it declares — and it must fire with the + // RELINKED paths, since those are the ones the renderer will ask for. + const seen: string[][] = []; + const service = new DocumentService(tempDir, mediaDir, (doc) => + seen.push(doc.assets.map((a) => a.originalPath)), + ); + const created = await service.createProject("P"); + const withAsset = await service.addAsset(created.project.id, { + path: path.join(mediaDir, "take.mp4"), + label: "take.mp4", + }); + seen.length = 0; + await service.getProject(created.project.id); + expect(seen).toEqual([withAsset.assets.map((a) => a.originalPath)]); + }); + + it("is optional, so a service built without it loads as it always did", async () => { + const created = await service.createProject("P"); + await expect(service.getProject(created.project.id)).resolves.toMatchObject({ + project: { id: created.project.id }, + }); + }); + }); + describe("addAsset", () => { it("appends a video asset and sets primaryAssetId on the first add", async () => { const doc = await service.createProject("P"); @@ -280,6 +308,63 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(first.project.primaryAssetId); expect(after.assets).toHaveLength(2); }); + + // Issue #350 — external audio import (voiceover / BGM / SFX). + it("appends an audio asset without claiming the primary slot", async () => { + const doc = await service.createProject("P"); + const updated = await service.addAsset(doc.project.id, { + path: "/tmp/voiceover.mp3", + kind: "audio", + }); + expect(updated.assets).toHaveLength(1); + expect(updated.assets[0]?.kind).toBe("audio"); + // An audio-only file must never become the project's primary asset, even + // when it is the first file added to an otherwise-empty project. + expect(updated.project.primaryAssetId).toBeUndefined(); + }); + + it("keeps the existing video primary when an audio track is added", async () => { + const doc = await service.createProject("P"); + const withVideo = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const primary = withVideo.project.primaryAssetId; + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/bgm.wav", + kind: "audio", + }); + expect(withAudio.project.primaryAssetId).toBe(primary); + expect(withAudio.assets).toHaveLength(2); + }); + + it("rejects unsupported audio extensions", async () => { + const doc = await service.createProject("P"); + await expect( + service.addAsset(doc.project.id, { path: "/tmp/clip.mp4", kind: "audio" }), + ).rejects.toBeInstanceOf(ProjectFileError); + }); + + it("accepts a recorded .webm take as audio", async () => { + // MediaRecorder writes a voiceover as webm/opus — the same extension a + // screen recording uses. The caller has already declared the kind here, + // so this gate must take it; only the import PICKER, which has nothing + // but the extension to go on, still refuses .webm as audio. + const doc = await service.createProject("P"); + const next = await service.addAsset(doc.project.id, { + path: "/tmp/voiceover-2026.webm", + kind: "audio", + }); + expect(next.assets.at(-1)).toMatchObject({ kind: "audio" }); + // ...and it must not have claimed the primary (video-only) slot. + expect(next.project.primaryAssetId).toBeUndefined(); + }); + + it("accepts a video extension under the default kind but not as audio", async () => { + const doc = await service.createProject("P"); + // The same extension routing works in reverse: an .mp3 is fine as audio + // but rejected as video (covered above), and an .mp4 is the opposite. + await expect( + service.addAsset(doc.project.id, { path: "/tmp/a.mp3", kind: "audio" }), + ).resolves.toBeDefined(); + }); }); describe("removeAsset", () => { @@ -363,6 +448,57 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(b.assets[1]?.id); }); + // Issue #350 — an audio overlay can never be primary. + it("passes primary to the next VIDEO asset, never to an audio asset", async () => { + const doc = await service.createProject("P"); + const video = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + await service.addAsset(doc.project.id, { path: "/tmp/music.mp3", kind: "audio" }); + const primaryId = video.project.primaryAssetId; + expect(primaryId).toBeTruthy(); + // Removing the only video leaves just the audio asset; primary must clear, + // not fall to the audio one. + const after = await service.removeAsset(doc.project.id, primaryId ?? ""); + expect(after.project.primaryAssetId).toBeUndefined(); + expect(after.assets).toHaveLength(1); + expect(after.assets[0]?.kind).toBe("audio"); + }); + + it("drops audioTracks that referenced a removed audio asset", async () => { + const doc = await service.createProject("P"); + await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/music.mp3", + kind: "audio", + }); + const audioId = withAudio.assets.find((a) => a.kind === "audio")?.id ?? ""; + expect(audioId).toBeTruthy(); + const withTrack = await service.saveProject({ + ...withAudio, + audioTracks: [ + { + id: "trk_1", + assetId: audioId, + kind: "music", + startMs: 0, + endMs: 10_000, + durationSec: 10, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "music", + origin: "user", + }, + ], + }); + expect(withTrack.audioTracks).toHaveLength(1); + const after = await service.removeAsset(doc.project.id, audioId); + expect(after.audioTracks).toEqual([]); + expect(after.assets.some((a) => a.id === audioId)).toBe(false); + }); + it("resequences other assets and rederives their anchored regions", async () => { const created = await service.createProject("P"); const withA = await service.addAsset(created.project.id, { path: "/tmp/a.mp4" }); diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 3c93e3bc0..e78a9c48e 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -21,6 +21,7 @@ import { documentSchema, migrateRawDocumentToCurrent, } from "../../src/lib/ai-edition/schema"; +import { ensureDocumentExtensions } from "../media/extensionClip"; import { relinkProjectMedia } from "../media/projectMediaRelinker"; const PROJECT_FILE_EXTENSION = ".openscreen"; @@ -38,6 +39,9 @@ export interface ProjectSummary { export interface AddAssetInput { path: string; label?: string; + // "audio" imports an external voiceover / BGM / SFX file (issue #350). + // Defaults to "video" when omitted, so existing callers are unaffected. + kind?: "video" | "audio"; } export class DocumentNotFoundError extends Error { @@ -72,6 +76,35 @@ function isSupportedVideoPath(filePath: string): boolean { return SUPPORTED_VIDEO_EXTENSIONS.has(ext); } +// Imported audio (issue #350). Decoding is handled downstream by the same +// WebCodecs / ffmpeg paths that read a video's audio track, so this list is the +// container formats decodeAudioData and the compositor can open. +// What may be filed as an AUDIO asset. Deliberately WIDER than the import +// picker's list in `electron/ipc/handlers.ts`: this gate runs when the caller +// has already declared `kind: "audio"`, so it only has to reject files that +// could not carry audio at all, whereas the picker has to guess from the +// extension alone and must not offer a video as audio. +// +// `.webm` is exactly that difference. An in-editor voiceover take is written by +// MediaRecorder as webm/opus — the same extension a screen recording uses — so +// the picker rightly refuses it while this gate must accept it. +const SUPPORTED_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".oga", + ".opus", + ".webm", +]); + +function isSupportedAudioPath(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase(); + return SUPPORTED_AUDIO_EXTENSIONS.has(ext); +} + function safeProjectId(raw: string): string { // ponytail: project ids are uuid-prefixed strings (e.g. "proj_"). Reject // anything that smells like path traversal before we ever touch the disk. @@ -124,9 +157,23 @@ export class DocumentService { // `mediaRegistryDir` is where the media-links registry file lives // (RECORDINGS_DIR in production) — see getProject. Injected for the same // reason as `projectsRoot`: this module stays free of any `electron` import. - constructor(projectsRoot: string, mediaRegistryDir: string) { + /** + * Called with every document this service hands out, so the process that owns the read + * allow-list can grant the media that document declares. + * + * Injected for the same reason as the two paths above: this module stays free of any + * `electron` import. Optional so the tests and the CLI construct it as they always did. + */ + private readonly onProjectRead?: (document: AxcutDocument) => void; + + constructor( + projectsRoot: string, + mediaRegistryDir: string, + onProjectRead?: (document: AxcutDocument) => void, + ) { this.projectsRoot = projectsRoot; this.mediaRegistryDir = mediaRegistryDir; + this.onProjectRead = onProjectRead; } async ensureProjectsDir(): Promise { @@ -241,7 +288,12 @@ export class DocumentService { // back, and it is not persisted from here: the renderer saves the document // it was given, as it does for any other load-time repair. const migrated = migrateRawDocumentToCurrent(JSON.parse(raw)); - return documentSchema.parse(await relinkProjectMedia(migrated, this.mediaRegistryDir)); + const document = documentSchema.parse( + await relinkProjectMedia(migrated, this.mediaRegistryDir), + ); + // AFTER the relink, so what is granted is the path the renderer will actually ask for. + this.onProjectRead?.(document); + return document; } async createProject(title: string): Promise { @@ -262,6 +314,9 @@ export class DocumentService { project: { ...parsed.project, updatedAt: new Date().toISOString() }, }; await this.writeProject(stamped); + // After the write, never before: a derived file is not worth delaying the user's edit + // reaching disk, and a failure to generate one must not fail the save. + await ensureDocumentExtensions(stamped); return stamped; } @@ -283,7 +338,15 @@ export class DocumentService { if (!input.path) { throw new ProjectFileError("Asset path is required.", projectId); } - if (!isSupportedVideoPath(input.path)) { + const kind = input.kind ?? "video"; + if (kind === "audio") { + if (!isSupportedAudioPath(input.path)) { + throw new ProjectFileError( + `Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`, + projectId, + ); + } + } else if (!isSupportedVideoPath(input.path)) { throw new ProjectFileError( `Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`, projectId, @@ -300,18 +363,24 @@ export class DocumentService { } const asset: AxcutAsset = { id: createId("asset"), - kind: "video", + kind, label: input.label?.trim() || path.basename(absolutePath), originalPath: absolutePath, sizeBytes, cameraTrack: null, }; + // An audio import is an overlay, never the thing the timeline is built + // around, so it must not claim the empty primaryAssetId slot — otherwise the + // first file dropped into a fresh project (a BGM track) would become its + // primary asset and the editor would try to lay out clips from a file with + // no video. + const claimsPrimary = kind !== "audio" && !doc.project.primaryAssetId; const next: AxcutDocument = { ...doc, assets: [...doc.assets, asset], project: { ...doc.project, - ...(doc.project.primaryAssetId ? {} : { primaryAssetId: asset.id }), + ...(claimsPrimary ? { primaryAssetId: asset.id } : {}), updatedAt: new Date().toISOString(), }, }; @@ -324,9 +393,13 @@ export class DocumentService { throw new ProjectFileError(`Asset ${assetId} not found in project ${projectId}.`, projectId); } const assets = doc.assets.filter((a) => a.id !== assetId); + // Primary is the thing the timeline is built around, so it must fall to the + // next VIDEO asset — never an audio overlay (issue #350), which can't be + // primary (see addAsset). Falling back to `assets[0]` would hand primary to + // an audio asset when the removed one was the last video. const primaryAssetId = doc.project.primaryAssetId === assetId - ? (assets[0]?.id ?? undefined) + ? (assets.find((a) => a.kind !== "audio")?.id ?? undefined) : doc.project.primaryAssetId; const withoutAssetClips = doc.timeline.clips .filter((clip) => clip.assetId === assetId) @@ -334,6 +407,9 @@ export class DocumentService { const next: AxcutDocument = { ...withoutAssetClips, assets, + // Drop imported audio tracks that referenced the removed asset — they + // would otherwise dangle, pointing at an asset the document no longer has. + audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId), timeline: { ...withoutAssetClips.timeline, trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId), diff --git a/electron/background-update.test.ts b/electron/background-update.test.ts new file mode 100644 index 000000000..53f92b0c5 --- /dev/null +++ b/electron/background-update.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it, vi } from "vitest"; +import { + planBackgroundUpdate, + runUnblockedDownloadAndInstall, + shouldQuitAndInstallAfterRestartPrompt, + shouldStartBackgroundUpdateTimer, +} from "./background-update"; + +describe("background update policy", () => { + it("starts no timer on a Store / non-owning channel", () => { + expect(shouldStartBackgroundUpdateTimer({ isPackaged: true, ownsItsUpdates: false })).toBe( + false, + ); + expect(shouldStartBackgroundUpdateTimer({ isPackaged: false, ownsItsUpdates: true })).toBe( + false, + ); + expect(shouldStartBackgroundUpdateTimer({ isPackaged: true, ownsItsUpdates: true })).toBe(true); + }); + + it("does not plan a current-version dialog on the background path", () => { + expect(planBackgroundUpdate({ outcome: { kind: "current" }, mode: "notify" })).toEqual({ + action: "none", + }); + expect( + planBackgroundUpdate({ outcome: { kind: "unsupported" }, mode: "download-and-install" }), + ).toEqual({ action: "none" }); + }); + + it("plans notify / download / download-and-install from an available update", () => { + const outcome = { kind: "downloaded" as const, version: "1.10.0" }; + expect(planBackgroundUpdate({ outcome, mode: "notify" })).toEqual({ + action: "notify-available", + version: "1.10.0", + }); + expect(planBackgroundUpdate({ outcome, mode: "download" })).toEqual({ + action: "download", + version: "1.10.0", + }); + expect(planBackgroundUpdate({ outcome, mode: "download-and-install" })).toEqual({ + action: "download-and-install", + version: "1.10.0", + }); + }); + + it("does not call quitAndInstall until Restart Now returns 0", async () => { + const install = vi.fn(); + const cancelled = await runUnblockedDownloadAndInstall({ + download: async () => ({ kind: "downloaded", version: "1.10.0" }), + blocked: () => null, + confirmRestart: async () => 1, + install, + }); + expect(cancelled).toEqual({ status: "cancelled" }); + expect(install).not.toHaveBeenCalled(); + expect(shouldQuitAndInstallAfterRestartPrompt(1)).toBe(false); + + const installed = await runUnblockedDownloadAndInstall({ + download: async () => ({ kind: "downloaded", version: "1.10.0" }), + blocked: () => null, + confirmRestart: async () => 0, + install, + }); + expect(installed).toEqual({ status: "installed" }); + expect(install).toHaveBeenCalledTimes(1); + expect(shouldQuitAndInstallAfterRestartPrompt(0)).toBe(true); + }); + + it("hands the download error back so the dialog can show its message", async () => { + const error = new Error("ECONNRESET mid-download"); + const failed = await runUnblockedDownloadAndInstall({ + download: async () => ({ kind: "failed", error }), + blocked: () => null, + confirmRestart: async () => 0, + install: vi.fn(), + }); + expect(failed).toEqual({ status: "failed", error }); + }); + + it("never reaches the restart prompt without a downloaded update", async () => { + // downloadSelfUpdate cannot return these today, but the type admits + // them; a current/unsupported outcome must not prompt or install. + for (const kind of ["current", "unsupported"] as const) { + const confirmRestart = vi.fn(async () => 0); + const install = vi.fn(); + const result = await runUnblockedDownloadAndInstall({ + download: async () => ({ kind }), + blocked: () => null, + confirmRestart, + install, + }); + expect(result).toEqual({ status: "unavailable" }); + expect(confirmRestart).not.toHaveBeenCalled(); + expect(install).not.toHaveBeenCalled(); + } + }); +}); diff --git a/electron/background-update.ts b/electron/background-update.ts new file mode 100644 index 000000000..ec17e3431 --- /dev/null +++ b/electron/background-update.ts @@ -0,0 +1,79 @@ +import type { UpdateOutcome } from "./auto-updater"; + +/** How far a background-discovered update may go on its own. No mode ever + * flips `autoInstallOnAppQuit`: install is always an explicit + * `quitAndInstall` behind a restart prompt, because `window-all-closed` + * quits this app and the HUD is a window — install-on-quit would fire a + * ~243 MB installer when the user merely closed the HUD. */ +export type UpdateMode = "notify" | "download" | "download-and-install"; + +export const DEFAULT_UPDATE_MODE: UpdateMode = "notify"; +export const BACKGROUND_UPDATE_INTERVAL_MS = 24 * 60 * 60 * 1000; + +export function parseUpdateMode(raw: unknown): UpdateMode { + if (raw === "notify" || raw === "download" || raw === "download-and-install") return raw; + return DEFAULT_UPDATE_MODE; +} + +export function shouldStartBackgroundUpdateTimer(input: { + isPackaged: boolean; + ownsItsUpdates: boolean; +}): boolean { + return input.isPackaged && input.ownsItsUpdates; +} + +export type BackgroundUpdatePlan = + | { action: "none" } + | { action: "notify-available"; version: string } + | { action: "download"; version: string } + | { action: "download-and-install"; version: string }; + +/** Background path: never plan a "you are current" dialog. */ +export function planBackgroundUpdate(input: { + outcome: UpdateOutcome; + mode: UpdateMode; +}): BackgroundUpdatePlan { + if (input.outcome.kind !== "downloaded") return { action: "none" }; + switch (input.mode) { + case "notify": + return { action: "notify-available", version: input.outcome.version }; + case "download": + return { action: "download", version: input.outcome.version }; + case "download-and-install": + return { action: "download-and-install", version: input.outcome.version }; + } +} + +export function shouldQuitAndInstallAfterRestartPrompt(response: number): boolean { + return response === 0; +} + +/** The failure carries the download error: the dialog that reports it shows + * `error.message` as detail, and collapsing the outcome to a bare string + * here is exactly how that detail once got lost between the helper and the + * caller. */ +export type DownloadAndInstallResult = + | { status: "failed"; error: Error } + | { status: "unavailable" } + | { status: "blocked" } + | { status: "cancelled" } + | { status: "installed" }; + +export async function runUnblockedDownloadAndInstall(deps: { + download: () => Promise; + blocked: () => string | null; + confirmRestart: () => Promise; + install: () => Promise; +}): Promise { + const downloaded = await deps.download(); + if (downloaded.kind === "failed") return { status: "failed", error: downloaded.error }; + // `downloadSelfUpdate` only ever reports downloaded|failed today, but the + // UpdateOutcome type admits current|unsupported — neither of which may + // reach the restart prompt, let alone quitAndInstall. + if (downloaded.kind !== "downloaded") return { status: "unavailable" }; + if (deps.blocked()) return { status: "blocked" }; + const choice = await deps.confirmRestart(); + if (!shouldQuitAndInstallAfterRestartPrompt(choice)) return { status: "cancelled" }; + await deps.install(); + return { status: "installed" }; +} diff --git a/electron/editorWindowState.test.ts b/electron/editorWindowState.test.ts new file mode 100644 index 000000000..a9dbcdcad --- /dev/null +++ b/electron/editorWindowState.test.ts @@ -0,0 +1,108 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + clampRectToWorkArea, + loadEditorWindowState, + resolveEditorCreation, + saveEditorWindowState, + shouldTrackEditorWindow, +} from "./editorWindowState"; + +const temps: string[] = []; + +afterEach(() => { + for (const dir of temps) rmSync(dir, { recursive: true, force: true }); + temps.length = 0; +}); + +function tmp(): string { + const dir = mkdtempSync(path.join(os.tmpdir(), "os-editor-win-")); + temps.push(dir); + return dir; +} + +describe("editor window state", () => { + it("returns null when the file is missing", () => { + expect(loadEditorWindowState(tmp())).toBeNull(); + }); + + it("clamps an off-screen rect onto the display workArea", () => { + const clamped = clampRectToWorkArea( + { x: -4000, y: 50, width: 1200, height: 800 }, + { x: 0, y: 0, width: 1920, height: 1080 }, + ); + expect(clamped.x).toBe(0); + expect(clamped.y).toBe(50); + expect(clamped.width).toBe(1200); + expect(clamped.height).toBe(800); + }); + + it("still maximizes when nothing is saved, and never loads or saves the bench", () => { + const missing = resolveEditorCreation({ isBench: false, saved: null }); + expect(missing.maximize).toBe(true); + expect(missing.persist).toBe(true); + expect(missing.bounds).toEqual({ width: 1200, height: 800 }); + + expect(shouldTrackEditorWindow({ windowType: "bench" })).toBe(false); + const bench = resolveEditorCreation({ isBench: true, saved: null }); + expect(bench.persist).toBe(false); + expect(bench.maximize).toBe(true); + + const dir = tmp(); + saveEditorWindowState(dir, { x: 10, y: 20, width: 1280, height: 720, maximized: false }); + expect(shouldTrackEditorWindow({ windowType: "bench" })).toBe(false); + expect(shouldTrackEditorWindow({})).toBe(true); + }); + + it("round-trips a save through load", () => { + const dir = tmp(); + const state = { x: 10, y: 20, width: 1280, height: 720, maximized: true }; + saveEditorWindowState(dir, state); + expect(loadEditorWindowState(dir)).toEqual(state); + }); + + it("returns null on a corrupted file instead of throwing", () => { + const dir = tmp(); + writeFileSync(path.join(dir, "editor-window.json"), "{ not json"); + expect(loadEditorWindowState(dir)).toBeNull(); + }); + + it("rejects garbage fields rather than restoring a broken rect", () => { + const dir = tmp(); + writeFileSync( + path.join(dir, "editor-window.json"), + JSON.stringify({ x: Number.NaN, y: 20, width: "1280", height: 720, maximized: false }), + ); + expect(loadEditorWindowState(dir)).toBeNull(); + }); + + it("rejects zero or negative dimensions instead of clamping them up", () => { + // A width:0 record is garbage the app never writes; letting it through + // would restore a min-size non-maximized window instead of the default. + for (const dims of [ + { width: 0, height: 720 }, + { width: 1280, height: 0 }, + { width: -1280, height: 720 }, + ]) { + const dir = tmp(); + writeFileSync( + path.join(dir, "editor-window.json"), + JSON.stringify({ x: 10, y: 20, ...dims, maximized: false }), + ); + expect(loadEditorWindowState(dir)).toBeNull(); + } + }); + + it("rejects a non-boolean maximized rather than coercing it", () => { + for (const maximized of ["true", 1, null, undefined]) { + const dir = tmp(); + writeFileSync( + path.join(dir, "editor-window.json"), + JSON.stringify({ x: 10, y: 20, width: 1280, height: 720, maximized }), + ); + expect(loadEditorWindowState(dir)).toBeNull(); + } + }); +}); diff --git a/electron/editorWindowState.ts b/electron/editorWindowState.ts new file mode 100644 index 000000000..6597e3736 --- /dev/null +++ b/electron/editorWindowState.ts @@ -0,0 +1,126 @@ +// Persist/restore the editor window's normal bounds and maximized flag. +// The export bench must never load or save this file — it measures a 1200×800 +// maximized window on purpose. + +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +export const DEFAULT_EDITOR_SIZE = { width: 1200, height: 800 }; +export const EDITOR_WINDOW_MIN = { width: 800, height: 600 }; + +export interface EditorWindowRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface EditorWindowState extends EditorWindowRect { + maximized: boolean; +} + +export interface DisplayWorkArea { + x: number; + y: number; + width: number; + height: number; +} + +export function editorWindowStatePath(userData: string): string { + return path.join(userData, "editor-window.json"); +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +export function parseEditorWindowState(raw: unknown): EditorWindowState | null { + if (!raw || typeof raw !== "object") return null; + const rec = raw as Record; + // Zero/negative dimensions and a non-boolean `maximized` are garbage the + // app never writes; restoring them would clamp into a small non-maximized + // window instead of the documented default. Reject the record whole. + if ( + !isFiniteNumber(rec.x) || + !isFiniteNumber(rec.y) || + !isFiniteNumber(rec.width) || + !isFiniteNumber(rec.height) || + rec.width <= 0 || + rec.height <= 0 || + typeof rec.maximized !== "boolean" + ) { + return null; + } + return { + x: rec.x, + y: rec.y, + width: rec.width, + height: rec.height, + maximized: rec.maximized, + }; +} + +export function loadEditorWindowState(userData: string): EditorWindowState | null { + const file = editorWindowStatePath(userData); + if (!existsSync(file)) return null; + try { + return parseEditorWindowState(JSON.parse(readFileSync(file, "utf8"))); + } catch { + return null; + } +} + +export function saveEditorWindowState(userData: string, state: EditorWindowState): void { + try { + writeFileSync(editorWindowStatePath(userData), `${JSON.stringify(state)}\n`, "utf8"); + } catch { + // Best-effort; a failed write must not block close. + } +} + +export function clampRectToWorkArea( + rect: EditorWindowRect, + workArea: DisplayWorkArea, + min = EDITOR_WINDOW_MIN, +): EditorWindowRect { + const width = Math.min(Math.max(rect.width, min.width), Math.max(min.width, workArea.width)); + const height = Math.min(Math.max(rect.height, min.height), Math.max(min.height, workArea.height)); + const maxX = workArea.x + Math.max(0, workArea.width - width); + const maxY = workArea.y + Math.max(0, workArea.height - height); + return { + x: Math.min(Math.max(rect.x, workArea.x), maxX), + y: Math.min(Math.max(rect.y, workArea.y), maxY), + width, + height, + }; +} + +export function shouldTrackEditorWindow(query: Record): boolean { + return query.windowType !== "bench"; +} + +export function resolveEditorCreation(input: { + isBench: boolean; + saved: EditorWindowState | null; +}): { + bounds: { x?: number; y?: number; width: number; height: number }; + maximize: boolean; + persist: boolean; +} { + if (input.isBench) { + return { bounds: { ...DEFAULT_EDITOR_SIZE }, maximize: true, persist: false }; + } + if (!input.saved) { + return { bounds: { ...DEFAULT_EDITOR_SIZE }, maximize: true, persist: true }; + } + return { + bounds: { + x: input.saved.x, + y: input.saved.y, + width: input.saved.width, + height: input.saved.height, + }, + maximize: input.saved.maximized, + persist: true, + }; +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 4eb288e14..6de7a3cc4 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -81,7 +81,10 @@ interface Window { requestNativeMacCursorAccess: () => Promise<{ success: boolean; granted: boolean; - status: string; + // "not-determined" is the only genuine denial; the rest mean the helper + // never got to ask. See macNativeCursorRecordingSession.ts. + status: "granted" | "not-determined" | "missing-helper" | "error" | "exited" | "timeout"; + accessibilityTrusted: boolean; error?: string; }>; assetBaseUrl: string; @@ -286,6 +289,22 @@ interface Window { name?: string; canceled?: boolean; }>; + // Import an external audio file from the timeline toolbar (issue #350). + openAudioFilePicker: () => Promise<{ + success: boolean; + path?: string; + name?: string; + canceled?: boolean; + message?: string; + }>; + // Persist an in-editor voiceover take (raw MediaRecorder bytes) under the + // recordings dir, so it outlives the session like every other asset. + saveRecordedVoiceover: (data: ArrayBuffer) => Promise<{ + success: boolean; + path?: string; + message?: string; + error?: string; + }>; setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>; setCurrentRecordingSession: ( session: import("../src/lib/recordingSession").RecordingSession | null, diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 8596dfafd..fcda39e0c 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -16,6 +16,7 @@ import { shell, systemPreferences, } from "electron"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; import { type NativeLinuxRecordingRequest, portalCursorMode, @@ -69,7 +70,10 @@ import { LinuxNativeCaptureSession, } from "../native-bridge/capture/linuxNativeCaptureSession"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; -import { requestMacCursorAccessibilityAccess } from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; +import { + isMacCursorHelperUnavailable, + requestMacCursorAccessibilityAccess, +} from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; import { findPipeWireCursorHelperPath } from "../native-bridge/cursor/recording/pipeWireCursorRecordingSession"; import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; import { toHelperRect } from "../native-bridge/helperCoordinates"; @@ -104,6 +108,9 @@ const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([ ".ts", ]); const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); +// See the save-recorded-voiceover handler: an upper bound on renderer-supplied +// bytes written to disk, well past any plausible take. +const MAX_RECORDED_VOICEOVER_BYTES = 512 * 1024 * 1024; const nativeMacCaptureEvents = new EventEmitter(); // Enumeration walks every display and window and grabs a thumbnail of each, so it @@ -183,6 +190,34 @@ function hasAllowedImportVideoExtension(filePath: string): boolean { return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); } +// Imported audio (issue #350). Kept separate from the video set so the two +// pickers stay honest — an audio picker must not approve a video path and vice +// versa. A SUBSET of SUPPORTED_AUDIO_EXTENSIONS in the document service, which +// also accepts `.webm`: that gate is told the kind by its caller, while this one +// only has the extension to go on and `.webm` is far more often a video. +const ALLOWED_IMPORT_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".opus", +]); + +function hasAllowedImportAudioExtension(filePath: string): boolean { + return ALLOWED_IMPORT_AUDIO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} + +// Video OR audio. The type-specific pickers stay honest (see the audio set's +// comment), but the generic media READS — peaks, binary, file-info, chunk — serve +// whichever kind the document points at, so they must accept both. Gating them on +// video alone dropped every imported audio path once `approvedPaths` was empty +// (a project reopen), and the waveform was lost for good (issue #350). +function hasAllowedImportMediaExtension(filePath: string): boolean { + return hasAllowedImportVideoExtension(filePath) || hasAllowedImportAudioExtension(filePath); +} + function runProcess( command: string, args: string[], @@ -279,8 +314,13 @@ async function prepareSupplementalPreviewAudioTrack(videoPath: string) { return { success: true, path: pathToFileURL(outputPath).toString() }; } -async function approveReadableVideoPath( - filePath?: string | null, +// Shared core behind the media path approvers. `hasAllowedExtension` is the ONLY +// thing that differs between video and audio imports, so it is the single knob: +// an already-approved path passes regardless, otherwise the extension gate, +// optional trusted-dir confinement, and a stat check decide whether to approve. +async function approveReadableMediaPath( + filePath: string | null | undefined, + hasAllowedExtension: (p: string) => boolean, trustedDirs?: string[], ): Promise { const normalizedPath = normalizeVideoSourcePath(filePath); @@ -292,7 +332,7 @@ async function approveReadableVideoPath( return normalizedPath; } - if (!hasAllowedImportVideoExtension(normalizedPath)) { + if (!hasAllowedExtension(normalizedPath)) { return null; } @@ -319,6 +359,53 @@ async function approveReadableVideoPath( return normalizedPath; } +function approveReadableVideoPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportVideoExtension, trustedDirs); +} + +function approveReadableAudioPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportAudioExtension, trustedDirs); +} + +/** + * A path a generic read may use — and NOT a way to obtain one. + * + * `approveReadableMediaPath` grants approval to any existing file with a media extension. + * Behind a picker or a document load that is the point; behind `read-binary-file` it meant + * the renderer could name any media file on the machine and have its bytes handed back, + * which is a capability no generic handler should carry (CWE-200). + * + * Approval is granted in exactly three places now: the recordings directory, a file the user + * picked, and the assets a loaded project declares (`approveDocumentMedia`). Everything else + * spends one. + */ +function readableApprovedPath(filePath?: string | null): string | null { + const normalizedPath = normalizeVideoSourcePath(filePath); + if (!normalizedPath) return null; + if (!isPathAllowed(normalizedPath)) return null; + // The extension check stays: an approval granted for a recording must not become a way + // to read the project file, the log, or anything else sitting beside it. + if (!hasAllowedImportMediaExtension(normalizedPath)) return null; + return normalizedPath; +} + +/** Grant the media a loaded project declares. The document is the app's own file, and this + * is what the picker's approval decays into once the app restarts. */ +function approveDocumentMedia(document: AxcutDocument): void { + for (const asset of document.assets ?? []) { + const media = normalizeVideoSourcePath(asset.originalPath); + if (media && hasAllowedImportMediaExtension(media)) approveFilePath(media); + const camera = normalizeVideoSourcePath(asset.cameraTrack?.sourcePath); + if (camera && hasAllowedImportMediaExtension(camera)) approveFilePath(camera); + } +} + function resolveRecordingOutputPath(fileName: string): string { const trimmed = fileName.trim(); if (!trimmed) { @@ -1353,6 +1440,12 @@ function readNativeWindowsEncoderSelection(output: string) { // which is what `salvageNativeWindowsFragmentedCapture` asks. container?: string; preferSoftwareEncoder?: boolean; + // Whether BeginWriting() actually landed on a hardware H.264 MFT, as + // opposed to `video` above, which only says which configuration path + // was tried. "default" plus a software runtime means the machine never + // got hardware acceleration in the first place -- see + // kVideoEncoderRuntime* in mf_encoder.h. + videoEncoderRuntime?: string; }; } catch { return null; @@ -1658,6 +1751,66 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ return { resolvedVia: "none" }; } +/** + * Writes the diagnostic bundle a bug report needs: app/OS facts, the native + * helpers' raw stdout/stderr (which is where `[stop-timing]` and + * `encoder-selection` land — see nativeWindowsCaptureStop.ts), and the main + * process's own recent console output. Shared by the renderer's IPC call and + * the menu/tray "Save Diagnostics" entry point in main.ts, which has no + * renderer-side `projectState`/`logs` to offer and does not need to. + */ +export async function exportDiagnosticFile(payload: { + error: string; + stack?: string; + projectState: unknown; + logs: string[]; +}) { + const { filePath, canceled } = await dialog.showSaveDialog({ + title: "Save Diagnostic File", + defaultPath: `openscreen-diagnostic-${Date.now()}.json`, + filters: [{ name: "JSON", extensions: ["json"] }], + }); + + if (canceled || !filePath) return { success: false, canceled: true }; + + const HELPER_OUTPUT_MAX_BYTES = 64 * 1024; + const tail = (s: string, max: number) => (s.length <= max ? s : s.slice(s.length - max)); + + const diagnostic = { + timestamp: new Date().toISOString(), + appVersion: app.getVersion(), + platform: process.platform, + arch: process.arch, + // The same fact the About box leads with, and for the same reason: it is what + // explains why a copy does or does not offer an update check. This file is the + // artifact users actually attach, so it must not be the one that omits it. + channel: getInstallChannel(), + osRelease: os.release(), + osVersion: os.version(), + totalMemoryMB: Math.round(os.totalmem() / 1024 / 1024), + nodeVersion: process.versions.node, + electronVersion: process.versions.electron, + chromeVersion: process.versions.chrome, + error: payload.error, + stack: payload.stack, + projectState: payload.projectState, + recentLogs: payload.logs, + helperOutput: { + windows: tail(nativeWindowsCaptureOutput, HELPER_OUTPUT_MAX_BYTES), + mac: tail(nativeMacCaptureOutput, HELPER_OUTPUT_MAX_BYTES), + }, + mainProcessLogs: mainLogBuffer.snapshot(), + }; + + try { + await fs.writeFile(filePath, JSON.stringify(diagnostic, null, 2), "utf-8"); + return { success: true, path: filePath }; + } catch (error) { + console.error("Failed to write diagnostic file:", error); + return { success: false, error: String(error) }; + } +} + export function registerIpcHandlers( createEditorWindow: () => void, createSourceSelectorWindow: () => BrowserWindow, @@ -1847,14 +2000,27 @@ export function registerIpcHandlers( ipcMain.handle("request-native-mac-cursor-access", async () => { const access = await requestMacCursorAccessibilityAccess(); - // When the editable cursor can't get Accessibility trust, pop a native dialog - // that deep-links to the Accessibility pane (mirrors the Screen Recording flow). + // Pop the native Accessibility dialog ONLY for a genuine denial — the helper ran, + // asked, and was told no. Every other !granted status means the helper never got + // to ask (absent from the build, killed by the loader, crashed, hung), and telling + // the user to grant a permission they may well already hold is what made #515 + // impossible to escape. Those degrade silently instead; the recorder falls back to + // position-only cursor telemetry and the countdown still runs. if (process.platform === "darwin" && !access.granted) { + if (isMacCursorHelperUnavailable(access.status)) { + console.warn( + `[cursor-macos] editable cursor unavailable (status=${access.status}${ + access.error ? `, error=${access.error}` : "" + }); the app ${ + access.accessibilityTrusted ? "does" : "does not" + } hold Accessibility trust. Recording continues with position-only cursor telemetry.`, + ); + return access; + } + const mainWin = getMainWindow(); const detail = - access.status === "missing-helper" - ? "The cursor helper couldn't be found in this build, so the editable cursor can't be enabled. Rebuild the native helper (npm run build:native:mac) or switch the HUD cursor mode to system." - : "Allow OpenScreen under System Settings → Privacy & Security → Accessibility, then press record again to start the countdown."; + "Allow OpenScreen under System Settings → Privacy & Security → Accessibility, then press record again to start the countdown."; const messageOptions = { type: "warning", buttons: ["Open Accessibility Settings", "Cancel"], @@ -2531,6 +2697,7 @@ export function registerIpcHandlers( path: outputPath, helperPath, videoEncoderSelection: encoderSelection?.video ?? null, + videoEncoderRuntime: encoderSelection?.videoEncoderRuntime ?? null, webcamUnavailable, microphoneDefaulted, }; @@ -3507,6 +3674,8 @@ export function registerIpcHandlers( } }); + // The media tab imports VIDEO (it arranges clips). Audio is imported from the + // timeline toolbar instead (issue #350) — see `open-audio-file-picker` below. ipcMain.handle("open-video-file-picker", async () => { try { const dialogOptions = buildDialogOptions( @@ -3553,6 +3722,84 @@ export function registerIpcHandlers( } }); + // Import an external audio file (voiceover / BGM / SFX) — issue #350. Driven by + // the timeline's "Add audio" tool: audio is a timeline overlay (like an + // annotation), not a media-tab clip, so it has its own audio-only picker and the + // renderer adds it as a kind:"audio" asset + track at the playhead. + ipcMain.handle("open-audio-file-picker", async () => { + try { + const dialogOptions = buildDialogOptions( + { + title: mainT("dialogs", "fileDialogs.selectAudio"), + defaultPath: RECORDINGS_DIR, + filters: [ + { + name: mainT("dialogs", "fileDialogs.audioFiles"), + extensions: ["mp3", "wav", "m4a", "aac", "flac", "ogg", "opus"], + }, + { name: mainT("dialogs", "fileDialogs.allFiles"), extensions: ["*"] }, + ], + properties: ["openFile"], + }, + getMainWindow(), + ); + const result = await dialog.showOpenDialog(dialogOptions); + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + + const normalizedPath = await approveReadableAudioPath(result.filePaths[0]); + if (!normalizedPath) { + return { + success: false, + message: "Selected file is not a supported readable audio file", + }; + } + + return { + success: true, + path: normalizedPath, + }; + } catch (error) { + console.error("Failed to open audio file picker:", error); + return { + success: false, + message: "Failed to open audio file picker", + error: String(error), + }; + } + }); + + // In-editor voiceover recording: the renderer hands over the raw MediaRecorder + // blob (webm/opus) and gets back the path it landed at, under the recordings + // dir so it lives with the project's other media and survives relaunches. + ipcMain.handle("save-recorded-voiceover", async (_event, data: ArrayBuffer) => { + try { + if (!(data instanceof ArrayBuffer) || data.byteLength === 0) { + return { success: false, message: "Empty recording" }; + } + // A cap, because this writes renderer-supplied bytes straight to disk. An + // hour of Opus is a few tens of MB, so 512 MB is far past any real take + // and still refuses a runaway or malformed payload before it is buffered. + if (data.byteLength > MAX_RECORDED_VOICEOVER_BYTES) { + return { success: false, message: "Recording too large" }; + } + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + const fileName = `voiceover-${new Date().toISOString().replace(/[:.]/g, "-")}.webm`; + const target = path.join(RECORDINGS_DIR, fileName); + await fs.writeFile(target, Buffer.from(data)); + return { success: true, path: target }; + } catch (error) { + console.error("Failed to save recorded voiceover:", error); + return { + success: false, + message: "Failed to save recorded voiceover", + error: String(error), + }; + } + }); + ipcMain.handle("reveal-in-folder", async (_, filePath: string) => { try { // showItemInFolder returns nothing, it throws on error @@ -3578,7 +3825,7 @@ export function registerIpcHandlers( ipcMain.handle("read-binary-file", async (_, filePath: string) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = readableApprovedPath(filePath); if (!normalizedPath) { return { success: false, @@ -3608,7 +3855,7 @@ export function registerIpcHandlers( // recording above that can never be loaded whole — see read-file-chunk). ipcMain.handle("get-readable-file-info", async (_, filePath: string) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = readableApprovedPath(filePath); if (!normalizedPath) { return { success: false, @@ -3644,7 +3891,7 @@ export function registerIpcHandlers( async (_, filePath: string, durationSec: number): Promise => { try { // Same approval gate as every other read of a renderer-supplied path. - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = readableApprovedPath(filePath); if (!normalizedPath) { return { success: false, message: "File path is not approved" }; } @@ -3668,7 +3915,7 @@ export function registerIpcHandlers( // do (2 GiB cap) and a 16 GB machine cannot hold for multi-GB recordings. ipcMain.handle("read-file-chunk", async (_, filePath: string, offset: number, length: number) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = readableApprovedPath(filePath); if (!normalizedPath) { return { success: false, @@ -4085,55 +4332,8 @@ export function registerIpcHandlers( ipcMain.handle( "save-diagnostic", - async ( - _, - payload: { error: string; stack?: string; projectState: unknown; logs: string[] }, - ) => { - const { filePath, canceled } = await dialog.showSaveDialog({ - title: "Save Diagnostic File", - defaultPath: `openscreen-diagnostic-${Date.now()}.json`, - filters: [{ name: "JSON", extensions: ["json"] }], - }); - - if (canceled || !filePath) return { success: false, canceled: true }; - - const HELPER_OUTPUT_MAX_BYTES = 64 * 1024; - const tail = (s: string, max: number) => (s.length <= max ? s : s.slice(s.length - max)); - - const diagnostic = { - timestamp: new Date().toISOString(), - appVersion: app.getVersion(), - platform: process.platform, - arch: process.arch, - // The same fact the About box leads with, and for the same reason: it is what - // explains why a copy does or does not offer an update check. This file is the - // artifact users actually attach, so it must not be the one that omits it. - channel: getInstallChannel(), - osRelease: os.release(), - osVersion: os.version(), - totalMemoryMB: Math.round(os.totalmem() / 1024 / 1024), - nodeVersion: process.versions.node, - electronVersion: process.versions.electron, - chromeVersion: process.versions.chrome, - error: payload.error, - stack: payload.stack, - projectState: payload.projectState, - recentLogs: payload.logs, - helperOutput: { - windows: tail(nativeWindowsCaptureOutput, HELPER_OUTPUT_MAX_BYTES), - mac: tail(nativeMacCaptureOutput, HELPER_OUTPUT_MAX_BYTES), - }, - mainProcessLogs: mainLogBuffer.snapshot(), - }; - - try { - await fs.writeFile(filePath, JSON.stringify(diagnostic, null, 2), "utf-8"); - return { success: true, path: filePath }; - } catch (error) { - console.error("Failed to write diagnostic file:", error); - return { success: false, error: String(error) }; - } - }, + async (_, payload: { error: string; stack?: string; projectState: unknown; logs: string[] }) => + exportDiagnosticFile(payload), ); // One instance each, not one per call. DocumentService serialises saves of a @@ -4144,6 +4344,7 @@ export function registerIpcHandlers( const aiEditionDocuments = new DocumentService( path.join(app.getPath("userData"), "projects"), RECORDINGS_DIR, + approveDocumentMedia, ); // LlmConfigStore is single-instance for a duller reason — its constructor does diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 47d66e270..7b01e0b2f 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -353,6 +353,12 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { return createSuccessResponse(requestId, { backend: compositorViewService.probeBackend(), }); + case "probeSegmentation": + // No view needed either: the layout panel decides whether to offer the + // camera-background control before any preview exists. + return createSuccessResponse(requestId, { + support: compositorViewService.probeSegmentation(), + }); case "setRect": compositorViewService.setRect(request.payload.id, request.payload.rect); return createSuccessResponse(requestId, { ok: true }); @@ -483,6 +489,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { request.payload.projectId, request.payload.path, request.payload.label, + request.payload.kind, ), ); case "document.removeAsset": diff --git a/electron/main.ts b/electron/main.ts index 85eb063b8..ec961aead 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -30,6 +30,13 @@ import { installSelfUpdate, type UpdateOutcome, } from "./auto-updater"; +import { + BACKGROUND_UPDATE_INTERVAL_MS, + planBackgroundUpdate, + runUnblockedDownloadAndInstall, + shouldStartBackgroundUpdateTimer, + type UpdateMode, +} from "./background-update"; import { parseCliArgs } from "./cli/args"; import { runCli } from "./cli/cliMain"; import { isDiagnosticModeEnabled, mainLogBuffer } from "./diagnostics/main-log-buffer"; @@ -40,11 +47,21 @@ import { unregisterAllGlobalShortcuts, } from "./globalShortcut"; import { mainT, setMainLocale } from "./i18n"; -import { getInstallChannel, offersUpdateCheck, platformOwnsUpdates } from "./install-channel"; -import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers"; +import { + getInstallChannel, + offersUpdateCheck, + ownsItsUpdates, + platformOwnsUpdates, +} from "./install-channel"; +import { + exportDiagnosticFile, + getSelectedDesktopSource, + registerIpcHandlers, +} from "./ipc/handlers"; import { installMainProcessErrorGuards } from "./main-process-errors"; import { registerSttIpc, shutdownStt } from "./stt"; import { checkLatestRelease } from "./update-checker"; +import { loadUpdateMode, saveUpdateMode } from "./update-settings"; import { createCountdownOverlayWindow, createEditorWindow, @@ -211,6 +228,11 @@ function setupApplicationMenu() { role: "about", label: mainT("common", "actions.about") || "About OpenScreen", }, + { type: "separator" as const }, + { + label: mainT("common", "actions.saveDiagnostics") || "Save Diagnostics", + click: runSaveDiagnostics, + }, // Omitted entirely — here, in the Help menu and in the tray — where a package // manager owns the update. See `canOfferUpdateCheck`. ...(canOfferUpdateCheck() @@ -369,6 +391,11 @@ function setupApplicationMenu() { label: mainT("common", "actions.about") || "About OpenScreen", click: runAboutDialog, }, + { type: "separator" as const }, + { + label: mainT("common", "actions.saveDiagnostics") || "Save Diagnostics", + click: runSaveDiagnostics, + }, ], }); } @@ -519,33 +546,108 @@ function runUpdateCheck() { }); } +/** + * Menu and tray entry point for exporting a diagnostic bundle. The backend + * (`exportDiagnosticFile`) and its "Save Diagnostics" label already existed — + * nothing in the app ever called it (getopenscreen/openscreen#460). Reveals + * the written file on success, the same confirmation the export flow's "Show + * in folder" gives, so there is no need for a second dialog on top of the + * native Save dialog the user already went through. + * + * No renderer `projectState`/`logs` to attach from here, unlike the in-app + * crash path this shares a payload shape with — the diagnostic value for a + * capture bug is almost entirely `helperOutput`/`mainProcessLogs`, which + * `exportDiagnosticFile` reads straight from the main process regardless. + */ +function runSaveDiagnostics() { + exportDiagnosticFile({ error: "Manual diagnostic export", projectState: null, logs: [] }) + .then((result) => { + if (result.canceled) return; + if (!result.success) { + // exportDiagnosticFile resolves rather than rejects on a write + // failure, so this is the branch that turns "user picked a save + // location and got silence" into a visible error instead of a + // menu action that looks like it did nothing. + showMessageBox({ + type: "error", + title: PRODUCT_NAME, + message: mainT("dialogs", "export.failed") || "Export Failed", + detail: result.error, + }).catch((error) => { + console.error("[diagnostics] failure dialog failed", error); + }); + return; + } + if (result.path) { + shell.showItemInFolder(result.path); + } + }) + .catch((error) => { + console.error("[diagnostics] save failed", error); + }); +} + /** Mirrors the flag that already drives the tray icon. An update must never interrupt a take — * and on Windows it physically cannot, because the capture helpers spawn from inside the * install directory and NSIS cannot overwrite a running .exe. */ let isRecording = false; +let currentUpdateMode: UpdateMode = "notify"; +let backgroundUpdateTimer: ReturnType | null = null; + +function showUpdateSettingsMenu(): boolean { + return app.isPackaged && ownsItsUpdates(getInstallChannel()); +} + +function persistUpdateMode(mode: UpdateMode) { + currentUpdateMode = mode; + saveUpdateMode(app.getPath("userData"), mode); + updateTrayMenu(isRecording); +} async function downloadAndInstall(latestVersion: string) { - const downloaded = await downloadSelfUpdate(); - if (downloaded.kind === "failed") { + const result = await runUnblockedDownloadAndInstall({ + download: downloadSelfUpdate, + blocked: () => + blockedFromInstalling({ + recording: isRecording, + inApplicationsFolder: + process.platform === "darwin" ? (app.isInApplicationsFolder?.() ?? true) : true, + platform: process.platform, + }), + confirmRestart: async () => { + const restart = await showMessageBox({ + type: "info", + title: PRODUCT_NAME, + message: mainT("common", "updates.readyToInstall", { latestVersion }), + buttons: [ + mainT("common", "actions.restartNow") || "Restart Now", + mainT("common", "actions.cancel") || "Cancel", + ], + defaultId: 0, + cancelId: 1, + }); + return restart.response; + }, + install: installSelfUpdate, + }); + if (result.status === "failed") { await showMessageBox({ type: "error", title: PRODUCT_NAME, // Not `updates.failed`: the CHECK succeeded — that is how we got here — and telling // the user we could not check for updates sends them looking in the wrong place. message: mainT("common", "updates.downloadFailed"), - detail: downloaded.error.message, + detail: result.error.message, }); return; } - - const blocked = blockedFromInstalling({ - recording: isRecording, - // macOS-only API; absent elsewhere, and irrelevant there. - inApplicationsFolder: - process.platform === "darwin" ? (app.isInApplicationsFolder?.() ?? true) : true, - platform: process.platform, - }); - if (blocked) { + if (result.status === "blocked") { + const blocked = blockedFromInstalling({ + recording: isRecording, + inApplicationsFolder: + process.platform === "darwin" ? (app.isInApplicationsFolder?.() ?? true) : true, + platform: process.platform, + }); await showMessageBox({ type: "info", title: PRODUCT_NAME, @@ -554,21 +656,78 @@ async function downloadAndInstall(latestVersion: string) { blocked === "recording" ? "updates.blockedRecording" : "updates.blockedLocation", ), }); - return; } +} - const restart = await showMessageBox({ +async function presentAvailableUpdate(latestVersion: string) { + const choice = await showMessageBox({ type: "info", title: PRODUCT_NAME, - message: mainT("common", "updates.readyToInstall", { latestVersion }), + message: mainT("common", "updates.available", { + currentVersion: app.getVersion(), + latestVersion, + }), buttons: [ - mainT("common", "actions.restartNow") || "Restart Now", + mainT("common", "actions.downloadUpdate") || "Download Update", mainT("common", "actions.cancel") || "Cancel", ], defaultId: 0, cancelId: 1, }); - if (restart.response === 0) await installSelfUpdate(); + if (choice.response === 0) await downloadAndInstall(latestVersion); +} + +async function runBackgroundUpdateCheck() { + if (updateCheckInFlight || !canOfferUpdateCheck()) return; + updateCheckInFlight = true; + try { + const outcome = await probeSelfUpdate(); + const plan = planBackgroundUpdate({ outcome, mode: currentUpdateMode }); + if (plan.action === "none") return; + if (plan.action === "notify-available") { + await presentAvailableUpdate(plan.version); + return; + } + if (plan.action === "download") { + const downloaded = await downloadSelfUpdate(); + if (downloaded.kind === "failed") { + await showMessageBox({ + type: "error", + title: PRODUCT_NAME, + message: mainT("common", "updates.downloadFailed"), + detail: downloaded.error.message, + }); + return; + } + await showMessageBox({ + type: "info", + title: PRODUCT_NAME, + message: mainT("common", "updates.downloaded", { latestVersion: plan.version }), + }); + return; + } + await downloadAndInstall(plan.version); + } catch (error) { + console.error("[updates] background check failed", error); + } finally { + updateCheckInFlight = false; + } +} + +function startBackgroundUpdateTimer() { + if (backgroundUpdateTimer) return; + if ( + !shouldStartBackgroundUpdateTimer({ + isPackaged: app.isPackaged, + ownsItsUpdates: ownsItsUpdates(getInstallChannel()), + }) + ) { + return; + } + backgroundUpdateTimer = setInterval(() => { + void runBackgroundUpdateCheck(); + }, BACKGROUND_UPDATE_INTERVAL_MS); + backgroundUpdateTimer.unref?.(); } /** `onVerdict` fires as soon as we know whether an update exists — before any of the dialogs @@ -716,6 +875,29 @@ function updateTrayMenu(recording: boolean = false) { }, ] : []), + ...(showUpdateSettingsMenu() + ? [ + { + label: mainT("common", "actions.updateSettings") || "Update Settings", + submenu: ( + [ + ["notify", "updateModeNotify", "Notify when an update is available"], + ["download", "updateModeDownload", "Download updates automatically"], + [ + "download-and-install", + "updateModeDownloadAndInstall", + "Download and install updates automatically", + ], + ] as const + ).map(([mode, key, fallback]) => ({ + label: mainT("common", `actions.${key}`) || fallback, + type: "radio" as const, + checked: currentUpdateMode === mode, + click: () => persistUpdateMode(mode), + })), + }, + ] + : []), // The About box's other homes are menu-bar items, and no window this app creates // shows a menu bar: the HUD is frameless (electron/windows.ts), and the editor and // notes windows call setAutoHideMenuBar(true) on Windows and Linux. Without this @@ -730,6 +912,14 @@ function updateTrayMenu(recording: boolean = false) { label: mainT("common", "actions.about") || "About OpenScreen", click: runAboutDialog, }, + // Right next to About, and reachable without opening any window: this is the + // one place in the app most likely to still be usable right after a recording + // failed to stop, which is exactly when the [stop-timing]/encoder-selection + // lines this exports are worth the most (getopenscreen/openscreen#460). + { + label: mainT("common", "actions.saveDiagnostics") || "Save Diagnostics", + click: runSaveDiagnostics, + }, { type: "separator" as const }, { label: mainT("common", "actions.quit") || "Quit", @@ -1073,8 +1263,14 @@ appReady?.then(async () => { }); }); + // Deliberately no updater touch here: importing electron-updater costs + // startup time and the channels that cannot use it must not pay for it at + // all (see auto-updater.ts getUpdater) — every real update path applies + // its settings lazily on first use. + currentUpdateMode = loadUpdateMode(app.getPath("userData")); createTray(); updateTrayMenu(); + startBackgroundUpdateTimer(); configureAboutPanel(); setupApplicationMenu(); await ensureRecordingsDir(); diff --git a/electron/media/extensionClip.test.ts b/electron/media/extensionClip.test.ts new file mode 100644 index 000000000..d2f7df681 --- /dev/null +++ b/electron/media/extensionClip.test.ts @@ -0,0 +1,74 @@ +// The one thing worth pinning: the command says what we mean. Running ffmpeg in a unit test +// would test ffmpeg, not us — the arguments are where a mistake actually lives. + +import { describe, expect, it } from "vitest"; +import { extensionClipPath } from "../../src/lib/ai-edition/timeline/clip-parts"; +import { extensionClipArgs } from "./extensionClip"; + +const SPEC = { durationSec: 3.6, fps: 30, width: 1920, height: 1080 }; + +describe("extensionClipArgs", () => { + const args = extensionClipArgs(SPEC, "C:/out/w1_3600.mp4"); + const filter = (prefix: string) => args.find((a) => a.startsWith(prefix)) ?? ""; + + it("draws a test pattern, so generated media is unmistakable on screen", () => { + // A held frame from the recording looked exactly like a decoder stuck at the end of + // a clip — which is the bug it hid for three rounds. + expect(filter("testsrc2=")).toContain("size=1920x1080"); + expect(filter("testsrc2=")).toContain("rate=30"); + }); + + it("carries an audible noise track rather than silence", () => { + expect(filter("anoisesrc=")).toContain("a=0.2"); + expect(args).toContain("1:a"); + }); + + it("reads the recording not at all — nothing to seek, nothing to decode", () => { + expect(args.filter((a) => a === "-i")).toHaveLength(2); + expect(args).not.toContain("-ss"); + expect(args.some((a) => a.endsWith(".mp4") && a !== "C:/out/w1_3600.mp4")).toBe(false); + }); + + it("runs for exactly the duration asked for, on both streams and the output", () => { + expect(filter("testsrc2=")).toContain("duration=3.600"); + expect(filter("anoisesrc=")).toContain("d=3.600"); + expect(args[args.indexOf("-t") + 1]).toBe("3.600"); + expect(args[args.length - 1]).toBe("C:/out/w1_3600.mp4"); + }); + + it("uses an encoder the bundled LGPL ffmpeg actually has", () => { + // `libx264` is GPL and absent: the first real run failed with "Unknown encoder". + expect(args).toContain("libopenh264"); + expect(args).not.toContain("libx264"); + }); + + it("still has a geometry when the asset does not know its own", () => { + // The live project's asset carries `fps: 0` — the probe never filled it in. + const blind = extensionClipArgs({ ...SPEC, fps: 0, width: 0, height: 0 }, "out.mp4"); + expect(blind.find((a) => a.startsWith("testsrc2="))).toContain("size=1920x1080"); + expect(blind.find((a) => a.startsWith("testsrc2="))).toContain("rate=30"); + }); +}); + +/** One backslash, built rather than escaped: the escape is what this test keeps losing. */ +const BS = String.fromCharCode(92); + +describe("extensionClipPath", () => { + it("sits beside the recording it was cut from, in a hidden folder", () => { + expect(extensionClipPath("C:/rec/take.mp4", "synth_2", 3.6)).toBe( + "C:/rec/.openscreen-extensions/synth_2_3600.mp4", + ); + }); + + it("carries the word and the duration, so a re-typed word asks for a different file", () => { + expect(extensionClipPath("C:/rec/take.mp4", "synth_2", 3.8)).not.toBe( + extensionClipPath("C:/rec/take.mp4", "synth_2", 3.6), + ); + }); + + it("is the same rule on a Windows path, so both processes name one file", () => { + expect(extensionClipPath(`C:${BS}rec${BS}take.mp4`, "w1", 1)).toBe( + `C:${BS}rec${BS}.openscreen-extensions${BS}w1_1000.mp4`, + ); + }); +}); diff --git a/electron/media/extensionClip.ts b/electron/media/extensionClip.ts new file mode 100644 index 000000000..d7641c25f --- /dev/null +++ b/electron/media/extensionClip.ts @@ -0,0 +1,167 @@ +/** + * The media an added word is spoken over. + * + * A word typed into the transcript has no recording behind it. Until there is TTS and frame + * generation the stand-in is a TEST PATTERN over noise — a real file, with real frames and a + * real audio track, so everything downstream decodes it like any other media instead of + * special-casing its absence. + * + * ponytail: a mire, on purpose, and not the recording's last frame held. A held frame is + * indistinguishable on screen from a decoder stuck at the end of a clip, which is exactly + * the bug it hid. The mire says "this is generated media, and it is playing HERE" at a + * glance. Swap it for synthesized frames the day there are any. + * + * DERIVED, never authored: the word is the truth, this file is regenerable from it. The name + * carries what it was generated from, so a stale one is simply never asked for again, and a + * missing one is a regeneration rather than a broken edit. + */ + +import { spawn } from "node:child_process"; +import { access, mkdir } from "node:fs/promises"; +import path from "node:path"; +import { isGeneratedAssetId } from "../../src/lib/ai-edition/document/insertion"; +import { resolveFfmpeg } from "./audioPeaks"; + +export interface ExtensionClipSpec { + durationSec: number; + /** Matched to the recording so the two concatenate without a re-encode downstream. + * `0` when the asset was imported before the probe filled it in — see the fallbacks. */ + fps: number; + width: number; + height: number; +} + +/** Noise rather than silence: a silent track is indistinguishable from a broken one, and + * this stands in for a voice that will be synthesized later. Loud enough to be unmistakable + * while the generated stretch is the thing being debugged. */ +const NOISE_AMPLITUDE = 0.2; +const SAMPLE_RATE = 48_000; + +/** The bundled ffmpeg is LGPL, so `libx264` is not in it — `libopenh264` is the software + * H.264 encoder every LGPL build carries, on every platform. */ +const VIDEO_ENCODER = "libopenh264"; + +/** Assets imported before the probe filled `video` carry zeroes, and the live project does. + * ponytail: fixed, read the real geometry off the source when the probe backfills it. */ +const FALLBACK_FPS = 30; +const FALLBACK_WIDTH = 1920; +const FALLBACK_HEIGHT = 1080; + +/** + * The ffmpeg arguments, as a pure function so the command can be asserted without running it. + * + * Two synthetic inputs and nothing else: the recording is not read at all, which is what + * makes this fast, independent of what the source codec is, and impossible to confuse with + * the recording once it is on screen. + */ +export function extensionClipArgs(spec: ExtensionClipSpec, outPath: string): string[] { + const dur = spec.durationSec.toFixed(3); + const fps = spec.fps > 0 ? spec.fps : FALLBACK_FPS; + const width = spec.width > 0 ? spec.width : FALLBACK_WIDTH; + const height = spec.height > 0 ? spec.height : FALLBACK_HEIGHT; + return [ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `testsrc2=size=${width}x${height}:rate=${fps}:duration=${dur}`, + "-f", + "lavfi", + "-i", + `anoisesrc=c=pink:a=${NOISE_AMPLITUDE}:r=${SAMPLE_RATE}:d=${dur}`, + "-map", + "0:v", + "-map", + "1:a", + "-c:v", + VIDEO_ENCODER, + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-t", + dur, + outPath, + ]; +} + +/** + * Every insertion's media, generated if it is not already there. + * + * Read off the ASSETS, not the words: an insertion is a clip on an asset that already knows + * its own path, its own length and its own geometry. There is nothing to derive here and + * nothing to agree with the renderer about beyond the path it stored. + * + * Called on SAVE, the only moment the main process — the one that can spawn ffmpeg — sees + * the document. Idempotent by name, so a save that adds nothing costs one `stat` per + * insertion. A failure is logged and swallowed: an edit is not lost because a derived file + * could not be written, and the clip renders black until the next save regenerates it. + */ +export async function ensureDocumentExtensions(document: { + assets: ReadonlyArray<{ + id: string; + originalPath?: string; + durationSec?: number; + video?: { width: number; height: number; fps: number }; + }>; +}): Promise { + for (const asset of document.assets) { + if (!isGeneratedAssetId(asset.id) || !asset.originalPath || !asset.durationSec) continue; + try { + await ensureExtensionClip( + { + durationSec: asset.durationSec, + fps: asset.video?.fps ?? 0, + width: asset.video?.width ?? 0, + height: asset.video?.height ?? 0, + }, + asset.originalPath, + ); + } catch (error) { + console.error(`[insertion] ${asset.id}: ${(error as Error).message}`); + } + } +} + +/** + * Generate the file if it is not already there, and return its path. + * + * Idempotent: the same word and duration name the same file, which is reused rather than + * re-encoded. The path is decided by `extensionClipPath`, so the renderer names the file it + * expects and this writes the file it named — one rule, both sides. + */ +export async function ensureExtensionClip( + spec: ExtensionClipSpec, + outPath: string, +): Promise { + try { + await access(outPath); + return outPath; + } catch { + // Not there yet — generate it. + } + const ffmpeg = resolveFfmpeg(); + if (!ffmpeg) throw new Error("no bundled ffmpeg to generate the extension clip with"); + await mkdir(path.dirname(outPath), { recursive: true }); + await run(ffmpeg, extensionClipArgs(spec, outPath)); + return outPath; +} + +function run(bin: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(bin, args, { stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr?.on("data", (chunk) => { + stderr += String(chunk); + }); + child.on("error", reject); + child.on("close", (code) => + code === 0 + ? resolve() + : reject(new Error(`ffmpeg exited ${code}${stderr ? `: ${stderr.trim()}` : ""}`)), + ); + }); +} diff --git a/electron/native-bridge/cursor/recording/macNativeCursorAccess.test.ts b/electron/native-bridge/cursor/recording/macNativeCursorAccess.test.ts new file mode 100644 index 000000000..a8a7dfd8c --- /dev/null +++ b/electron/native-bridge/cursor/recording/macNativeCursorAccess.test.ts @@ -0,0 +1,205 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * The cast on `actual` is written out in the factory rather than shared in a + * helper: `vi.mock` calls are HOISTED above every top-level statement, so a + * module-scope helper is still in its temporal dead zone when the factory runs. + */ +type WithDefault = { default?: Record }; + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + const spawn = vi.fn(); + return { ...actual, spawn, default: { ...((actual as WithDefault).default ?? {}), spawn } }; +}); + +const mocks = vi.hoisted(() => ({ + isTrustedAccessibilityClient: vi.fn(() => true), + // Shared rather than two separate vi.fn()s so a test can make every candidate path + // unreadable and reach the missing-helper branch. + accessSync: vi.fn(), +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + // No helper binary exists in a test checkout; by default pretend the first candidate + // path is executable so path resolution is not what is under test. + return { + ...actual, + accessSync: mocks.accessSync, + default: { ...((actual as WithDefault).default ?? {}), accessSync: mocks.accessSync }, + }; +}); + +vi.mock("electron", () => ({ + systemPreferences: { isTrustedAccessibilityClient: mocks.isTrustedAccessibilityClient }, + screen: { + getCursorScreenPoint: () => ({ x: 0, y: 0 }), + getDisplayNearestPoint: () => ({ scaleFactor: 2 }), + }, +})); + +import { spawn } from "node:child_process"; +import { + isMacCursorHelperUnavailable, + requestMacCursorAccessibilityAccess, +} from "./macNativeCursorRecordingSession"; + +/** Minimal stand-in for the cursor helper: stdio pipes plus kill bookkeeping. */ +class FakeHelper extends EventEmitter { + stdout = new PassThrough(); + stderr = new PassThrough(); + killed = false; + + kill() { + this.killed = true; + return true; + } + + /** Feeds one NDJSON line, the way the real helper emits them. */ + emitEvent(event: Record) { + this.stdout.write(`${JSON.stringify(event)}\n`); + } +} + +const spawnMock = vi.mocked(spawn); +let helper: FakeHelper; +let originalPlatform: PropertyDescriptor | undefined; + +beforeEach(() => { + originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "darwin", configurable: true }); + helper = new FakeHelper(); + spawnMock.mockReset(); + spawnMock.mockReturnValue(helper as unknown as ReturnType); + mocks.isTrustedAccessibilityClient.mockReset(); + mocks.isTrustedAccessibilityClient.mockReturnValue(true); + mocks.accessSync.mockReset(); + const silence = () => { + // The access probe logs every helper diagnostic; keep the test output readable. + }; + vi.spyOn(console, "warn").mockImplementation(silence); + vi.spyOn(console, "error").mockImplementation(silence); +}); + +afterEach(() => { + if (originalPlatform) { + Object.defineProperty(process, "platform", originalPlatform); + } + vi.restoreAllMocks(); +}); + +/** Lets the spawn listeners attach before the fake helper speaks. */ +async function settle(pending: Promise, act: () => void): Promise { + await Promise.resolve(); + act(); + return pending; +} + +describe("requestMacCursorAccessibilityAccess", () => { + it("grants when the helper reports Accessibility trust", async () => { + const access = await settle(requestMacCursorAccessibilityAccess(), () => + helper.emitEvent({ type: "ready", timestampMs: 1, accessibilityTrusted: true }), + ); + + expect(access).toMatchObject({ success: true, granted: true, status: "granted" }); + }); + + it("reports a genuine denial when the helper ran and was told no", async () => { + const access = await settle(requestMacCursorAccessibilityAccess(), () => + helper.emitEvent({ type: "ready", timestampMs: 1, accessibilityTrusted: false }), + ); + + expect(access).toMatchObject({ granted: false, status: "not-determined" }); + // The ONLY status that should ever raise the "grant Accessibility" dialog. + expect(isMacCursorHelperUnavailable(access.status)).toBe(false); + }); + + /** + * The regression test for #515. On macOS 12 the helper was stamped with a macOS 13 + * deployment target, so it died in the loader before printing its `ready` line — and + * the app answered by telling the user to grant a permission they already held. + * A helper that never got to ask must never be reported as a denial. + */ + it("does not call a helper that died before ready a denied permission", async () => { + const access = await settle(requestMacCursorAccessibilityAccess(), () => + helper.emit("exit", null, "SIGABRT"), + ); + + expect(access.granted).toBe(false); + expect(access.status).toBe("exited"); + // The app itself IS trusted — proof this is a broken build, not a missing grant. + expect(access.accessibilityTrusted).toBe(true); + expect(isMacCursorHelperUnavailable(access.status)).toBe(true); + }); + + it("distinguishes a helper that could not be spawned at all", async () => { + const access = await settle(requestMacCursorAccessibilityAccess(), () => + helper.emit("error", new Error("spawn ENOENT")), + ); + + expect(access).toMatchObject({ granted: false, status: "error" }); + expect(isMacCursorHelperUnavailable(access.status)).toBe(true); + }); + + it("distinguishes a helper that hung without ever answering", async () => { + vi.useFakeTimers(); + try { + const pending = requestMacCursorAccessibilityAccess(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(5_000); + const access = await pending; + + expect(access).toMatchObject({ granted: false, status: "timeout" }); + expect(isMacCursorHelperUnavailable(access.status)).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + /** + * The other half of #515's conflation, and the branch whose dialog used to tell the + * user to run a build script. No helper on disk is not a permission problem either. + */ + it("reports an absent helper as unavailable, not as a denial", async () => { + mocks.accessSync.mockImplementation(() => { + throw new Error("ENOENT"); + }); + mocks.isTrustedAccessibilityClient.mockReturnValue(false); + + const access = await requestMacCursorAccessibilityAccess(); + + expect(access).toMatchObject({ success: true, granted: false, status: "missing-helper" }); + expect(access.accessibilityTrusted).toBe(false); + expect(isMacCursorHelperUnavailable(access.status)).toBe(true); + // Nothing was spawned: there was nothing to spawn. + expect(spawnMock).not.toHaveBeenCalled(); + }); + + /** + * The probe must not raise the macOS Accessibility prompt. It runs before the helper + * is even located, so on every unavailable branch it would be asking for a grant that + * is not what is missing. + */ + it("reads Accessibility trust without prompting", async () => { + await settle(requestMacCursorAccessibilityAccess(), () => + helper.emitEvent({ type: "ready", timestampMs: 1, accessibilityTrusted: true }), + ); + + expect(mocks.isTrustedAccessibilityClient).toHaveBeenCalledWith(false); + expect(mocks.isTrustedAccessibilityClient).not.toHaveBeenCalledWith(true); + }); + + it("keeps the app's own trust separate from the helper's fate", async () => { + mocks.isTrustedAccessibilityClient.mockReturnValue(false); + + const access = await settle(requestMacCursorAccessibilityAccess(), () => + helper.emit("exit", 1, null), + ); + + expect(access.accessibilityTrusted).toBe(false); + expect(access.status).toBe("exited"); + }); +}); diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts index e274b681f..a8d916a59 100644 --- a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -80,98 +80,142 @@ export function findMacCursorHelperPath() { return null; } -export async function requestMacCursorAccessibilityAccess() { +/** + * Why `granted: false` is not the same question as "did the user deny Accessibility". + * + * `not-determined` is the ONLY genuine denial: the helper ran, asked, and was told no. + * The other four mean the helper never got to ask — it is absent from the build, the + * loader killed it, it crashed, or it hung. Reporting those as a denial is what made + * #515 unfixable from the user's side: on macOS 12 the helper died in dyld, and the app + * answered by telling the user to grant a permission they had already granted. + */ +export type MacCursorAccessStatus = + | "granted" + | "not-determined" + | "missing-helper" + | "error" + | "exited" + | "timeout"; + +export interface MacCursorAccessResult { + success: boolean; + granted: boolean; + status: MacCursorAccessStatus; + /** + * Whether *the app* holds Accessibility trust, read from the main process rather + * than from the helper. This is what separates the two failure modes: a helper that + * could not run while this is `true` is a broken build, not a missing grant. + */ + accessibilityTrusted: boolean; + error?: string; +} + +/** True when the helper never got far enough to answer the permission question. */ +export function isMacCursorHelperUnavailable(status: MacCursorAccessStatus) { + return ( + status === "missing-helper" || status === "error" || status === "exited" || status === "timeout" + ); +} + +export async function requestMacCursorAccessibilityAccess(): Promise { if (process.platform !== "darwin") { - return { success: true, granted: true, status: "granted" }; + return { success: true, granted: true, status: "granted", accessibilityTrusted: true }; } + // The return value is the signal, not a side effect: it says whether OpenScreen.app + // itself is trusted, independently of whether the child helper can be launched. + // + // `false`, so this is a silent read. Prompting here would ask for Accessibility + // BEFORE discovering whether the helper can run at all — and in every branch below + // where it cannot (missing-helper, error, exited, timeout) the grant is not what is + // missing, so the prompt is exactly the noise this function now exists to stop. + // + // Nothing is lost on the one path that does ask the user for the grant: reaching + // `not-determined` means the helper RAN, and it calls AXIsProcessTrustedWithOptions + // with kAXTrustedCheckOptionPrompt itself on every start + // (OpenScreenMacOSCursorHelper/main.swift), which is what puts OpenScreen in the + // Accessibility list for the user to tick. + let accessibilityTrusted = false; try { - systemPreferences.isTrustedAccessibilityClient(true); + accessibilityTrusted = systemPreferences.isTrustedAccessibilityClient(false); } catch { - // Continue with helper probing; it can trigger the same macOS prompt. + // Continue with helper probing; the helper performs the same check itself. } const helperPath = findMacCursorHelperPath(); if (!helperPath) { - return { success: true, granted: false, status: "missing-helper" }; + return { success: true, granted: false, status: "missing-helper", accessibilityTrusted }; } - return new Promise<{ success: boolean; granted: boolean; status: string; error?: string }>( - (resolve) => { - const child = spawn(helperPath, [JSON.stringify({ sampleIntervalMs: 250 })], { - stdio: ["ignore", "pipe", "pipe"], + return new Promise((resolve) => { + const child = spawn(helperPath, [JSON.stringify({ sampleIntervalMs: 250 })], { + stdio: ["ignore", "pipe", "pipe"], + }); + let settled = false; + let lineBuffer = ""; + const finish = (result: Omit) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + if (!child.killed) { + child.kill("SIGTERM"); + } + resolve({ ...result, accessibilityTrusted }); + }; + const timer = setTimeout(() => { + finish({ + success: false, + granted: false, + status: "timeout", + error: "Timed out waiting for macOS cursor helper", }); - let settled = false; - let lineBuffer = ""; - const finish = (result: { - success: boolean; - granted: boolean; - status: string; - error?: string; - }) => { - if (settled) { - return; - } - settled = true; - clearTimeout(timer); - if (!child.killed) { - child.kill("SIGTERM"); - } - resolve(result); - }; - const timer = setTimeout(() => { - finish({ - success: false, - granted: false, - status: "timeout", - error: "Timed out waiting for macOS cursor helper", - }); - }, READY_TIMEOUT_MS); + }, READY_TIMEOUT_MS); - child.stdout.setEncoding("utf8"); - child.stdout.on("data", (chunk: string) => { - lineBuffer += chunk; - const lines = lineBuffer.split(/\r?\n/); - lineBuffer = lines.pop() ?? ""; - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed) { - continue; - } - try { - const event = JSON.parse(trimmed) as MacCursorEvent; - if (event.type === "ready") { - finish({ - success: true, - granted: event.accessibilityTrusted === true, - status: event.accessibilityTrusted === true ? "granted" : "not-determined", - }); - return; - } - } catch { - // Ignore non-JSON helper output. + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + lineBuffer += chunk; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + try { + const event = JSON.parse(trimmed) as MacCursorEvent; + if (event.type === "ready") { + finish({ + success: true, + granted: event.accessibilityTrusted === true, + status: event.accessibilityTrusted === true ? "granted" : "not-determined", + }); + return; } + } catch { + // Ignore non-JSON helper output. } - }); + } + }); - child.once("error", (error) => { - finish({ - success: false, - granted: false, - status: "error", - error: error.message, - }); + child.once("error", (error) => { + finish({ + success: false, + granted: false, + status: "error", + error: error.message, }); - child.once("exit", (code, signal) => { - finish({ - success: false, - granted: false, - status: "exited", - error: `macOS cursor helper exited before ready (code=${code}, signal=${signal})`, - }); + }); + child.once("exit", (code, signal) => { + finish({ + success: false, + granted: false, + status: "exited", + error: `macOS cursor helper exited before ready (code=${code}, signal=${signal})`, }); - }, - ); + }); + }); } function normalizeCursorType(value: unknown): NativeCursorType | null { @@ -204,6 +248,10 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { this.previousLeftButtonDown = false; this.consecutiveOutsideSamples = 0; + // `true` here, unlike the silent read in requestMacCursorAccessibilityAccess: the + // return value is discarded, so prompting IS the point. Recording is starting and + // the helper is about to spawn, so this is the moment the grant can still change + // what the take records. try { systemPreferences.isTrustedAccessibilityClient(true); } catch { diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts index 69ae80e3a..d0b93adff 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts @@ -56,13 +56,25 @@ describe("PipeWireCursorAccumulator", () => { expect(point.timeMs).toBe(500); }); - it("reports every sample as a move, because Wayland exposes no buttons", () => { + it("defaults a sample with no interaction to a move", () => { + // The helper omits interactionType on the common case, so the accumulator + // owns the "move" fallback. This is what the user not being in the `input` + // group looks like: every sample arrives bare. const accumulator = new PipeWireCursorAccumulator(100); accumulator.reset(0); accumulator.addSample(sample(10, 5, 5)); expect(accumulator.toRecordingData().samples[0].interactionType).toBe("move"); }); + it("preserves a click the helper read from evdev", () => { + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(0); + accumulator.addSample(sample(10, 5, 5, { interactionType: "click" })); + accumulator.addSample(sample(20, 6, 6)); + const { samples } = accumulator.toRecordingData(); + expect(samples.map((s) => s.interactionType)).toEqual(["click", "move"]); + }); + it("re-bases onto the video's start and drops what came before it", () => { // This is the single-session case. Cursor samples start flowing as soon // as the helper does, but the video's frame 0 is only stamped once the diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts index 84a13130e..4fa434093 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts @@ -57,6 +57,11 @@ export type PipeWireHelperEvent = visible: boolean; assetId?: string; asset?: PipeWireCursorAssetPayload; + /** `"click"` on the sample coinciding with a left-button press the + * helper read from evdev; absent on a plain move (see the helper's + * input.rs). The helper never emits the `"move"` default — that word + * is filled in below so it lives in exactly one place. */ + interactionType?: "move" | "click"; } | { event: "audio-source"; @@ -166,8 +171,10 @@ export class PipeWireCursorAccumulator { cx: clamp(payload.x / width, 0, 1), cy: clamp(payload.y / height, 0, 1), visible: payload.visible, - // Wayland exposes no click events to an unprivileged process. - interactionType: "move", + // The portal never reports a button; the helper tags a sample "click" + // only when it read a left-button press from evdev (needs the user in + // the `input` group). Everything else — the common case — is a move. + interactionType: payload.interactionType ?? "move", ...(payload.assetId ? { assetId: payload.assetId } : {}), }); diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts index 16815071c..7dd222ced 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts @@ -21,8 +21,11 @@ import type { CursorRecordingSession } from "./session"; * * Two consequences the caller should know about: * - * * `interactionType` is always "move". Wayland exposes no portal for mouse - * buttons and /dev/input/event* is root:input, so clicks are unobtainable. + * * `interactionType` is "move" unless the helper could read left-button + * presses from evdev — which needs the user in the `input` group, because + * Wayland exposes no portal for mouse buttons and /dev/input/event* is + * root:input. When it can, the coinciding sample is tagged "click"; when it + * cannot, every sample is a move, as before. See the helper's input.rs. * * The helper raises its own portal picker. On Wayland, Electron's * `desktopCapturer` already raised one, so the user currently picks a source * twice. Merging the two is the job of the capture stage that will reuse this diff --git a/electron/native-bridge/services/aiEditionService.ts b/electron/native-bridge/services/aiEditionService.ts index 0fbbccc9c..90088781a 100644 --- a/electron/native-bridge/services/aiEditionService.ts +++ b/electron/native-bridge/services/aiEditionService.ts @@ -151,9 +151,17 @@ export class AiEditionService { } } - async addAsset(projectId: string, path: string, label?: string): Promise { - const document = await this.options.documents.addAsset(projectId, { path, label }); - const assetId = document.project.primaryAssetId ?? document.assets.at(-1)?.id ?? ""; + async addAsset( + projectId: string, + path: string, + label?: string, + kind?: "video" | "audio", + ): Promise { + const document = await this.options.documents.addAsset(projectId, { path, label, kind }); + // The just-added asset is always the last one; primaryAssetId is only a + // fallback for the video case and would point at the wrong asset for an + // audio import (which never claims primary), so prefer the tail. + const assetId = document.assets.at(-1)?.id ?? document.project.primaryAssetId ?? ""; return { assetId, document }; } diff --git a/electron/native-bridge/services/compositorViewService.test.ts b/electron/native-bridge/services/compositorViewService.test.ts index 1569816af..29f492b10 100644 --- a/electron/native-bridge/services/compositorViewService.test.ts +++ b/electron/native-bridge/services/compositorViewService.test.ts @@ -259,6 +259,9 @@ describe("resolveSceneAssetPaths", () => { resources = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-scene-assets-")); fs.mkdirSync(path.join(resources, "wallpapers"), { recursive: true }); fs.writeFileSync(path.join(resources, "wallpapers", "wallpaper1.jpg"), "jpg"); + const modelDir = path.join(resources, "mediapipe", "selfie_segmentation"); + fs.mkdirSync(modelDir, { recursive: true }); + fs.writeFileSync(path.join(modelDir, "selfie_segmentation_landscape.onnx"), "onnx"); const assetPaths = [ ...Object.values(themed?.assets ?? {}).map((a) => a.assetPath), ...Object.values(DEFAULT_CURSOR_SPRITES).map((s) => s.assetPath), @@ -299,6 +302,34 @@ describe("resolveSceneAssetPaths", () => { * service declares for the sprite map it builds. */ type ResolvedSprite = { path: string; hotspotX: number; hotspotY: number }; + // The renderer asks for an effect and knows nothing about the disk; this process answers + // where the model is. Same division as the wallpaper and the cursor sprites above. + it("fills in the segmentation model path when the scene asks for an effect", () => { + const out = resolved({ webcamEffect: { mode: "blur", blurIntensity: 0.5 } }); + expect(out.webcamEffect.modelPath).toBe( + path.join( + resources, + "mediapipe", + "selfie_segmentation", + "selfie_segmentation_landscape.onnx", + ), + ); + }); + + it("leaves the model path alone when no effect is requested", () => { + expect(resolved({ webcamEffect: { mode: "none" } }).webcamEffect.modelPath).toBeUndefined(); + expect(resolved({ background: { kind: "color", color: "#000" } }).webcamEffect).toBeUndefined(); + }); + + // A model that does not resolve must turn the effect off in the compositor, not fail the + // scene — the same contract a missing cursor sprite has. + it("leaves the model path unset rather than inventing one when the file is absent", () => { + fs.rmSync(path.join(resources, "mediapipe"), { recursive: true, force: true }); + const out = resolved({ webcamEffect: { mode: "transparent" } }); + expect(out.webcamEffect.modelPath).toBeUndefined(); + expect(out.webcamEffect.mode).toBe("transparent"); + }); + it("resolves a bundled wallpaper to the extraResources copy, not the unreadable asar path", () => { const out = resolved({ background: { kind: "image", path: "/wallpapers/wallpaper1.jpg" } }); @@ -307,6 +338,57 @@ describe("resolveSceneAssetPaths", () => { expect(fs.existsSync(out.background.path)).toBe(true); }); + // The camera's own background under the "custom" mode. It was NOT resolved, and the failure + // was silent and total: the compositor got "/wallpapers/wallpaper1.jpg", could not open it, + // and painted the bubble black — behind every one of the 18 bundled wallpapers, including + // the default. The screen's background had the fix; this one was simply missed. + it("resolves the camera's custom background, not just the screen's", () => { + const out = resolved({ + webcamEffect: { + mode: "custom", + background: { kind: "image", path: "/wallpapers/wallpaper1.jpg" }, + }, + }); + + expect(out.webcamEffect.background.path).toBe( + path.join(resources, "wallpapers", "wallpaper1.jpg"), + ); + expect(out.webcamEffect.background.path).not.toContain("app.asar"); + expect(fs.existsSync(out.webcamEffect.background.path)).toBe(true); + }); + + // The two are independent: a scene can put a wallpaper behind the screen and a different one + // behind the camera, and resolving one must not depend on the other being present. + it("resolves both backgrounds in the same scene", () => { + const out = resolved({ + background: { kind: "image", path: "/wallpapers/wallpaper1.jpg" }, + webcamEffect: { + mode: "custom", + background: { kind: "image", path: "/wallpapers/wallpaper1.jpg" }, + }, + }); + + expect(out.background.path).toBe(path.join(resources, "wallpapers", "wallpaper1.jpg")); + expect(out.webcamEffect.background.path).toBe(out.background.path); + }); + + // A colour or a gradient carries no path, and the compositor renders both itself. Rewriting + // them would be a bug, not a no-op. + it("leaves a colour or gradient camera background untouched", () => { + const colour = resolved({ + webcamEffect: { mode: "custom", background: { kind: "color", color: "#ff0080" } }, + }); + expect(colour.webcamEffect.background).toEqual({ kind: "color", color: "#ff0080" }); + + const gradient = resolved({ + webcamEffect: { + mode: "custom", + background: { kind: "gradient", angleDeg: 90, stops: ["#000", "#fff"] }, + }, + }); + expect(gradient.webcamEffect.background.stops).toEqual(["#000", "#fff"]); + }); + it("resolves a cursor theme's arrow sprite to a path that exists on disk", () => { if (!themed) return; // no bundled theme ships an arrow override const arrow = resolved({ cursor: { theme: themed.id } }).cursor.cursorSprites.arrow; diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 9efc302ec..59d0ee547 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -16,6 +16,7 @@ import type { GifParamsInput, NativeFramePacket, RemuxStats, + SegmentationSupport, } from "../../native/compositor-view/addon"; /** @@ -114,6 +115,11 @@ function resolveCursorSpritePaths( return resolved; } +/** Where the segmentation model sits under `public/`, and therefore under `dist/` once Vite + * has copied it. Resolved here rather than in the renderer: the compositor runs in this + * process, and the renderer has no business knowing the on-disk layout. */ +const SEGMENTATION_MODEL_ASSET = "mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx"; + export function resolveSceneAssetPaths(sceneJson: string): string { try { const scene = JSON.parse(sceneJson) as { @@ -122,21 +128,50 @@ export function resolveSceneAssetPaths(sceneJson: string): string { theme?: string; cursorSprites?: Record; }; + webcamEffect?: { + mode?: string; + modelPath?: string; + background?: { kind?: string; path?: string }; + }; }; let changed = false; - const bg = scene.background; - if (bg?.kind === "image" && typeof bg.path === "string" && bg.path.startsWith("/")) { + // Both backgrounds go through this: the screen's, and the camera's under the "custom" + // mode. The camera one was missed, and the failure is silent — the compositor gets + // "/wallpapers/wallpaper1.jpg", `image::open` cannot find it, and the PiP falls back to + // a flat colour with only a line on stderr to say so. + const resolveBackgroundImage = (target?: { kind?: string; path?: string }): boolean => { + if ( + target?.kind !== "image" || + typeof target.path !== "string" || + !target.path.startsWith("/") + ) { + return false; + } // strip the leading slash so path.join keeps it under the base dir - const resolved = resolveSceneAssetPath(bg.path.replace(/^\/+/, "")); - if (resolved) { - bg.path = resolved; - changed = true; + const resolved = resolveSceneAssetPath(target.path.replace(/^\/+/, "")); + if (!resolved) { + return false; } - } + target.path = resolved; + return true; + }; + changed = resolveBackgroundImage(scene.background) || changed; + changed = resolveBackgroundImage(scene.webcamEffect?.background) || changed; if (scene.cursor && typeof scene.cursor.theme === "string") { scene.cursor.cursorSprites = resolveCursorSpritePaths(scene.cursor.theme); changed = true; } + // The scene asks for an effect; this process says where the model is. A model that + // does not resolve leaves `modelPath` unset, which turns the effect off in the + // compositor rather than failing the scene — same contract as a missing cursor sprite. + const effect = scene.webcamEffect; + if (effect && typeof effect.mode === "string" && effect.mode !== "none") { + const resolved = resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET); + if (resolved) { + effect.modelPath = resolved; + changed = true; + } + } return changed ? JSON.stringify(scene) : sceneJson; } catch { return sceneJson; @@ -335,6 +370,37 @@ function ensureFfmpegSharedDllsOnPath(appRoot: string): void { process.env.PATH = `${dir}${path.delimiter}${current}`; } +/** The ONNX Runtime shared library's file name for this platform. */ +function ortLibName(): string { + if (process.platform === "win32") return "onnxruntime.dll"; + if (process.platform === "darwin") return "libonnxruntime.dylib"; + return "libonnxruntime.so"; +} + +/** + * Points `ORT_DYLIB_PATH` at the staged ONNX Runtime, which the addon loads dynamically for + * the webcam segmentation mask. + * + * It lives in the same arch-tagged `electron/native/bin//` directory the addon itself + * ships from, next to the ffmpeg DLLs — the convention `whisper-stt` already established for + * native sidecars. The crate links `ort` with `load-dynamic`, so the library is resolved at + * runtime rather than at build time: absent, `Segmenter::load` fails, the compositor logs one + * line and draws the webcam unsegmented. That is why this is best-effort and never throws. + */ +function ensureOnnxRuntimeOnPath(appRoot: string): void { + if (process.env.ORT_DYLIB_PATH) { + return; + } + const lib = ortLibName(); + for (const dir of ffmpegSharedBinCandidates(appRoot)) { + const candidate = path.join(dir, lib); + if (fs.existsSync(candidate)) { + process.env.ORT_DYLIB_PATH = candidate; + return; + } + } +} + function tryLoadAddon(candidates: string[]): CompositorViewAddon | null { for (const candidate of candidates) { try { @@ -382,6 +448,7 @@ export class CompositorViewService { const isPackaged = this.options.isPackaged ?? defaultIsPackaged(); ensureFfmpegSharedDllsOnPath(appRoot); + ensureOnnxRuntimeOnPath(appRoot); const candidates = buildCandidatePaths(appRoot, isPackaged, envOverride); const loaded = tryLoadAddon(candidates); if (!loaded) { @@ -419,6 +486,36 @@ export class CompositorViewService { } } + /** Whether this machine can actually segment the camera, and if not, what is missing. + * + * Three things have to line up, and each of them has been silently absent at some point: + * the addon, the ONNX Runtime library, and the model. The renderer used to guess from + * `process.platform`, which was wrong in both directions — it hid the control on Linux + * builds that could segment, and shows it on Intel Macs, for which upstream publishes no + * ONNX binary at all. A dev checkout and a `--dir` build have none staged either. + * + * Same shape as `probeBackend`: asked without allocating a view, because the panel needs + * the answer before any preview exists. */ + probeSegmentation(): SegmentationSupport { + const addon = this.ensureAddon(); + if (!addon) { + return "none"; + } + try { + if (!addon.segmentationRuntimeAvailable()) { + return "no-runtime"; + } + } catch (err) { + // An older `.node` predates this probe. Treat as unsupported rather than crashing + // the bridge — same contract as `probeBackend`. + console.warn("[compositor-view] segmentationRuntimeAvailable unavailable:", err); + return "none"; + } + // The model is resolved by this process, not the addon, so it is checked here — and it + // is the same lookup `resolveSceneAssetPaths` performs, so the two cannot disagree. + return resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET) ? "ready" : "no-model"; + } + /** Allocates an offscreen compositor view sized to `rect.width`x`rect.height`. * `rect.x` / `rect.y` are vestigial (ignored native-side) — the renderer * keeps them on the wire so the existing `CompositorViewRect` shape stays diff --git a/electron/native/README.md b/electron/native/README.md index 8ff2e3cdd..9bd69ae81 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -209,9 +209,14 @@ electron/native/bin/linux-x64/openscreen-pipewire-helper '{"probeOnly":true}' ### Known gaps -- **Mouse clicks are unobtainable.** Wayland exposes no portal for input events - and `/dev/input/event*` is `root:input`, so every sample's `interactionType` is - `"move"`. +- **Mouse clicks need the `input` group.** Wayland exposes no portal for input + events, so the helper reads left-button presses straight from evdev + (`/dev/input/event*`). Those nodes are `root:input`, so a user outside the + `input` group gets no readable device and every sample's `interactionType` + stays `"move"` — the same as before. When a device is readable, the coinciding + sample is tagged `"click"`. Scope is deliberately narrow: `BTN_LEFT` only, + never keystrokes (see `pipewire-capture/src/input.rs`), and + `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` turns it off entirely. - **The user picks a source twice.** Electron's `desktopCapturer` raises its own portal dialog for the video, and this helper raises a second one for the cursor. Collapsing them requires one portal session serving both, which is why the diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index dc5fe599a..4c63065d8 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -104,11 +104,22 @@ export interface ClipInput { * all, so the view will fail with its own, more specific message. */ export type CompositorBackend = "hardware" | "cpu" | "none"; +/** Whether this machine can segment the camera, and if not, what is missing. Mirrored in + * `src/native/contracts.ts` for the renderer, the same way `CompositorBackend` is — the addon + * boundary and the IPC boundary each own their shape. */ +export type SegmentationSupport = "ready" | "no-runtime" | "no-model" | "none"; + export interface CompositorViewAddon { /** What this machine offers, asked without allocating a view — the export dialog * needs the answer before any preview exists. Cached native-side. */ probeBackend(): CompositorBackend; + /** Whether the ONNX Runtime library is where the app staged it, and therefore whether a + * segmentation mask can be produced at all. Asked of the machine rather than guessed from + * the platform: upstream publishes no ONNX build for Intel Macs, and a dev checkout or a + * `--dir` build has none either. */ + segmentationRuntimeAvailable(): boolean; + /** Allocates an offscreen compositor view sized to `rect.width`x`rect.height` (the * target preview resolution; `rect.x` / `rect.y` are vestigial and ignored native-side). * No HWND/native-window-handle is passed: there's no OS window to parent to. The diff --git a/electron/native/pipewire-capture/Cargo.lock b/electron/native/pipewire-capture/Cargo.lock index f8fa27a58..17c2abd0a 100644 --- a/electron/native/pipewire-capture/Cargo.lock +++ b/electron/native/pipewire-capture/Cargo.lock @@ -238,6 +238,18 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -415,6 +427,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "evdev" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b686663ba7f08d92880ff6ba22170f1df4e83629341cba34cf82cd65ebea99" +dependencies = [ + "bitvec", + "cfg-if", + "libc", + "nix", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -475,6 +499,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.33" @@ -834,6 +864,7 @@ dependencies = [ "base64", "bindgen", "cc", + "evdev", "png", "pollster", "serde", @@ -974,6 +1005,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.7" @@ -1213,6 +1250,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.27.0" @@ -1463,6 +1506,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xdg-home" version = "1.3.0" diff --git a/electron/native/pipewire-capture/Cargo.toml b/electron/native/pipewire-capture/Cargo.toml index 604da16af..862b848bd 100644 --- a/electron/native/pipewire-capture/Cargo.toml +++ b/electron/native/pipewire-capture/Cargo.toml @@ -50,6 +50,7 @@ serde_json = "1" png = "0.17" base64 = "0.22" sha2 = "0.10" +evdev = "0.13" [build-dependencies] cc = "1" diff --git a/electron/native/pipewire-capture/build.rs b/electron/native/pipewire-capture/build.rs index 43ff9d6d4..d6aec8e88 100644 --- a/electron/native/pipewire-capture/build.rs +++ b/electron/native/pipewire-capture/build.rs @@ -24,7 +24,7 @@ fn main() { fn build_pipewire_shim(root: &Path) { let vendor = root.join("vendor/pipewire-1.0.5/include"); - let sources = ["csrc/pw_shim.c", "csrc/pw_audio.c"]; + let sources = ["csrc/pw_shim.c", "csrc/pw_audio.c", "csrc/dmabuf_modifiers.c"]; assert!( vendor.join("pipewire/pipewire.h").is_file(), @@ -127,7 +127,9 @@ fn link_ffmpeg(root: &Path) { ); println!("cargo:rustc-link-search=native={}", lib.display()); - for name in ["avcodec", "avformat", "avutil", "swscale", "swresample"] { + // avfilter is for the VAAPI VPP (scale_vaapi) that converts an imported + // dmabuf surface to NV12 for the encoder — see the dmabuf import path. + for name in ["avcodec", "avformat", "avutil", "avfilter", "swscale", "swresample"] { println!("cargo:rustc-link-lib={name}"); } // A SUBDIRECTORY, NOT `$ORIGIN`. The helper is staged into @@ -167,8 +169,12 @@ fn link_ffmpeg(root: &Path) { #include #include #include + #include #include #include + #include + #include + #include #include #include "#, @@ -181,6 +187,9 @@ fn link_ffmpeg(root: &Path) { .allowlist_function("avcodec_.*") .allowlist_function("avformat_.*") .allowlist_function("avio_.*") + .allowlist_function("avfilter_.*") + .allowlist_function("av_buffersrc_.*") + .allowlist_function("av_buffersink_.*") .allowlist_function("sws_.*") .allowlist_function("swr_.*") .allowlist_type("AV.*") diff --git a/electron/native/pipewire-capture/csrc/dmabuf_modifiers.c b/electron/native/pipewire-capture/csrc/dmabuf_modifiers.c new file mode 100644 index 000000000..96f1849c8 --- /dev/null +++ b/electron/native/pipewire-capture/csrc/dmabuf_modifiers.c @@ -0,0 +1,94 @@ +#include "dmabuf_modifiers.h" + +#include +#include + +/* + * Minimal EGL surface, spelled out rather than pulled from so the + * build needs no EGL dev package (same reasoning as the DRM modifier constants + * in pw_shim.c). libEGL itself is dlopen'd at runtime; if it is absent the + * caller degrades to the LINEAR/INVALID offer. + */ +typedef void *EGLDisplay; +typedef unsigned int EGLBoolean; +typedef int EGLint; +typedef intptr_t EGLAttrib; +typedef uint64_t EGLuint64KHR; + +#define OSC_EGL_TRUE 1 +#define OSC_EGL_NO_DISPLAY ((EGLDisplay)0) +#define OSC_EGL_DEFAULT_DISPLAY ((void *)0) +/* EGL_MESA_platform_surfaceless — a display with no window system, exactly what + * a one-shot capability query wants. */ +#define OSC_EGL_PLATFORM_SURFACELESS_MESA 0x31DD + +typedef void *(*osc_eglGetProcAddress)(const char *); +typedef EGLDisplay (*osc_eglGetPlatformDisplay)(EGLint platform, void *native, + const EGLAttrib *attrib_list); +typedef EGLBoolean (*osc_eglInitialize)(EGLDisplay, EGLint *major, EGLint *minor); +typedef EGLBoolean (*osc_eglTerminate)(EGLDisplay); +typedef EGLBoolean (*osc_eglQueryDmaBufModifiersEXT)(EGLDisplay, EGLint format, + EGLint max_modifiers, + EGLuint64KHR *modifiers, + EGLBoolean *external_only, + EGLint *num_modifiers); + +int osc_query_dmabuf_modifiers(uint32_t fourcc, uint64_t *out, int max_out) +{ + if (out == NULL || max_out <= 0) { + return 0; + } + + /* RTLD_NODELETE: EGL keeps process-global state, so never let dlclose run + * its destructors — we deliberately do not dlclose at all. */ + void *egl = dlopen("libEGL.so.1", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE); + if (egl == NULL) { + return 0; + } + + osc_eglGetProcAddress get_proc = + (osc_eglGetProcAddress)dlsym(egl, "eglGetProcAddress"); + osc_eglInitialize egl_init = (osc_eglInitialize)dlsym(egl, "eglInitialize"); + osc_eglTerminate egl_terminate = (osc_eglTerminate)dlsym(egl, "eglTerminate"); + if (get_proc == NULL || egl_init == NULL || egl_terminate == NULL) { + return 0; + } + + osc_eglGetPlatformDisplay get_display = + (osc_eglGetPlatformDisplay)get_proc("eglGetPlatformDisplayEXT"); + osc_eglQueryDmaBufModifiersEXT query_mods = + (osc_eglQueryDmaBufModifiersEXT)get_proc("eglQueryDmaBufModifiersEXT"); + if (get_display == NULL || query_mods == NULL) { + return 0; + } + + EGLDisplay dpy = get_display(OSC_EGL_PLATFORM_SURFACELESS_MESA, + OSC_EGL_DEFAULT_DISPLAY, NULL); + if (dpy == OSC_EGL_NO_DISPLAY) { + return 0; + } + if (egl_init(dpy, NULL, NULL) != OSC_EGL_TRUE) { + return 0; + } + + int written = 0; + EGLint count = 0; + if (query_mods(dpy, (EGLint)fourcc, 0, NULL, NULL, &count) == OSC_EGL_TRUE && + count > 0) { + EGLuint64KHR mods[128]; + EGLBoolean external[128]; + EGLint cap = (EGLint)(sizeof(mods) / sizeof(mods[0])); + if (count > cap) { + count = cap; + } + if (query_mods(dpy, (EGLint)fourcc, count, mods, external, &count) == + OSC_EGL_TRUE) { + for (EGLint i = 0; i < count && written < max_out; i++) { + out[written++] = (uint64_t)mods[i]; + } + } + } + + egl_terminate(dpy); + return written; +} diff --git a/electron/native/pipewire-capture/csrc/dmabuf_modifiers.h b/electron/native/pipewire-capture/csrc/dmabuf_modifiers.h new file mode 100644 index 000000000..be6ac1212 --- /dev/null +++ b/electron/native/pipewire-capture/csrc/dmabuf_modifiers.h @@ -0,0 +1,20 @@ +#ifndef OSC_DMABUF_MODIFIERS_H +#define OSC_DMABUF_MODIFIERS_H + +#include + +/* + * Query the DRM format modifiers the local GPU's EGL stack can import for a + * given DRM fourcc (e.g. XRGB8888). These are the modifiers we can legitimately + * advertise to the compositor in the dmabuf EnumFormat: a compositor buffer + * whose modifier is in this set is one we can hand to VAAPI. Fills `out` with up + * to `max_out` modifiers and returns the count, or 0 when enumeration is + * unavailable (no libEGL, no surfaceless platform, driver refuses) — in which + * case the caller falls back to LINEAR/INVALID only. + * + * libEGL is loaded with dlopen, matching how this crate treats libpipewire: the + * helper stays buildable and runnable on a box without EGL dev packages. + */ +int osc_query_dmabuf_modifiers(uint32_t fourcc, uint64_t *out, int max_out); + +#endif diff --git a/electron/native/pipewire-capture/csrc/pw_shim.c b/electron/native/pipewire-capture/csrc/pw_shim.c index 82a8a129e..d0ff4ce64 100644 --- a/electron/native/pipewire-capture/csrc/pw_shim.c +++ b/electron/native/pipewire-capture/csrc/pw_shim.c @@ -45,6 +45,7 @@ #include #include "pw_shim.h" +#include "dmabuf_modifiers.h" /* Defined next to osc_map_dmabuf; used earlier, at format negotiation. */ static int osc_debug_enabled(void); @@ -61,16 +62,43 @@ static int osc_debug_enabled(void); * header would put libdrm-dev in the build path of every contributor and CI * runner for two integers. That is the same trade the dlopen above makes. * - * These two are the ONLY modifiers this helper advertises, and the reason is - * osc_map_dmabuf(): a linear or implicit buffer can be read through a plain - * mmap of the dmabuf fd, while a tiled or compression-enabled one cannot — its - * bytes are not in raster order, so handing them to the encoder would produce a - * scrambled recording rather than an error. Anything else needs a real GPU - * import (EGL/gbm), which this helper deliberately does not link. + * LINEAR and INVALID are the universal fallbacks: a linear or implicit buffer + * can be read through a plain mmap of the dmabuf fd (osc_map_dmabuf). Tiled or + * compression-enabled buffers cannot — their bytes are not in raster order — so + * they require a real GPU import, which is being added for the VAAPI path (see + * issue #507 and docs/dmabuf-vaapi-plan.md). The additional importable modifiers + * are enumerated at runtime via EGL (osc_query_dmabuf_modifiers). */ #define OSC_DRM_FORMAT_MOD_LINEAR 0ULL #define OSC_DRM_FORMAT_MOD_INVALID 0x00ffffffffffffffULL +/* DRM fourccs for the 32-bit RGB formats we offer. XRGB8888 = fourcc('X','R', + * '2','4'); the others follow the same little-endian spelling. Used to enumerate + * importable modifiers and to describe a dmabuf to the GPU importer. Spelled out + * for the same reason as the modifiers above. */ +#define OSC_DRM_FORMAT_XRGB8888 0x34325258u /* SPA BGRx */ +#define OSC_DRM_FORMAT_ARGB8888 0x34325241u /* SPA BGRA */ +#define OSC_DRM_FORMAT_XBGR8888 0x34324258u /* SPA RGBx */ +#define OSC_DRM_FORMAT_ABGR8888 0x34324241u /* SPA RGBA */ + +/* SPA video format (byte order B,G,R,x ...) → the matching DRM fourcc (a + * little-endian 32-bit word), for the GPU dmabuf import. 0 = unmapped. */ +static uint32_t osc_spa_format_to_drm_fourcc(uint32_t spa_format) +{ + switch (spa_format) { + case SPA_VIDEO_FORMAT_BGRx: + return OSC_DRM_FORMAT_XRGB8888; + case SPA_VIDEO_FORMAT_BGRA: + return OSC_DRM_FORMAT_ARGB8888; + case SPA_VIDEO_FORMAT_RGBx: + return OSC_DRM_FORMAT_XBGR8888; + case SPA_VIDEO_FORMAT_RGBA: + return OSC_DRM_FORMAT_ABGR8888; + default: + return 0; + } +} + /* * Mapped dmabuf fds, keyed by fd. * @@ -86,6 +114,11 @@ static int osc_debug_enabled(void); */ #define OSC_MAX_DMABUF_MAPS 32 +/* The negotiated buffer pool is at most 16 (SPA_PARAM_BUFFERS below), plus a + * transient overlap while a renegotiation swaps the set. 32 covers it with room + * to spare, and a full table only means a held buffer is treated as stale. */ +#define OSC_MAX_LIVE_BUFFERS 32 + struct osc_dmabuf_map { int fd; void *ptr; @@ -179,14 +212,38 @@ struct osc_pw_session { struct spa_video_info_raw format; int buffer_info_reports; int want_video; + /* Set by the caller when the VAAPI dmabuf-import pipeline is available, which + * makes the stream offer dmabuf BEFORE shm so a tiled monitor buffer is + * imported on the GPU instead of copied through throttled shm (issue #507). + * shm stays in the offer as the fallback, so a compositor that cannot produce + * dmabuf still negotiates. */ + int prefer_dmabuf; /* Set from the negotiated format's SPA_VIDEO_FLAG_MODIFIER, which is what * decides whether buffers arrive as dmabuf fds or shared memory. */ int uses_dmabuf; + /* Latched when a dmabuf buffer cannot be CPU-mmap'd (a tiled buffer on, e.g., + * AMD/mutter). Frames then travel as raw dmabuf descriptors for a GPU import + * (issue #507) instead of the shared-memory path. */ + int import_dmabuf; struct osc_dmabuf_map dmabuf_maps[OSC_MAX_DMABUF_MAPS]; /* fd whose DMA_BUF_SYNC_START has not been closed by its END yet, or -1. * The bracket has to span the on_frame callback, not just osc_read_frame, * because the callback is where the pixels are actually read. */ int dmabuf_sync_fd; + /* Every pw_buffer the stream currently owns, added in osc_on_add_buffer and + * cleared in osc_on_remove_buffer. A dmabuf frame the consumer holds for a + * GPU import (issue #507) keeps the pw_buffer pointer, but a renegotiation + * destroys the buffer set — so osc_pw_requeue_buffer must check the handle is + * still here before touching it, or it would queue freed storage. + * + * Pointer equality alone is not enough: PipeWire reuses these wrapper slots, + * so a renegotiation can register a NEW buffer at the SAME address as one the + * consumer still holds. Each registration therefore carries a unique + * `generation`, and a retained handle is only re-queued when BOTH the pointer + * and its generation match — an ABA guard. `next_generation` never repeats. */ + struct pw_buffer *live_buffers[OSC_MAX_LIVE_BUFFERS]; + uint64_t live_generations[OSC_MAX_LIVE_BUFFERS]; + uint64_t next_generation; }; struct osc_pw_audio_api osc_audio_api; @@ -394,7 +451,8 @@ static const struct spa_pod *osc_build_enum_format(struct spa_pod_builder *build * which needs a GPU query, so letting the producer fixate is both simpler and * one fewer round trip that can go wrong. */ -static const struct spa_pod *osc_build_enum_format_dmabuf(struct spa_pod_builder *builder) +static const struct spa_pod *osc_build_enum_format_dmabuf(struct spa_pod_builder *builder, + int prefer_dmabuf) { struct spa_pod_frame object_frame; struct spa_pod_frame choice_frame; @@ -415,9 +473,29 @@ static const struct spa_pod *osc_build_enum_format_dmabuf(struct spa_pod_builder * tolerating the key. */ spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY); spa_pod_builder_push_choice(builder, &choice_frame, SPA_CHOICE_Enum, 0); - /* Default first, then every alternative — the default is repeated, same + /* Advertise the modifiers our GPU's EGL can import, so a tiled compositor + * buffer — the common case on AMD/mutter — negotiates as dmabuf instead of + * falling back to the throttled shm path (issue #507). LINEAR and INVALID + * stay as universal fallbacks. Modifiers match across the 32-bit RGB formats + * we offer, so enumerating XRGB8888 is representative. + * + * ONLY when prefer_dmabuf: the tiled modifiers are advertised solely when the + * VAAPI import pipeline is available. Otherwise a producer that offers no shm + * format (some wlroots/portal setups) could select a tiled buffer we cannot + * read, where before it would have fallen to a CPU-mappable LINEAR/INVALID + * dmabuf. Offering just those two keeps that path intact. + * + * Default first, then every alternative — the default is repeated, same * idiom as SPA_POD_CHOICE_ENUM_Id above. */ - spa_pod_builder_long(builder, (int64_t)OSC_DRM_FORMAT_MOD_LINEAR); + uint64_t egl_mods[128]; + int egl_mod_count = + prefer_dmabuf ? osc_query_dmabuf_modifiers(OSC_DRM_FORMAT_XRGB8888, egl_mods, 128) : 0; + int64_t default_mod = + egl_mod_count > 0 ? (int64_t)egl_mods[0] : (int64_t)OSC_DRM_FORMAT_MOD_LINEAR; + spa_pod_builder_long(builder, default_mod); + for (int i = 0; i < egl_mod_count; i++) { + spa_pod_builder_long(builder, (int64_t)egl_mods[i]); + } spa_pod_builder_long(builder, (int64_t)OSC_DRM_FORMAT_MOD_LINEAR); spa_pod_builder_long(builder, (int64_t)OSC_DRM_FORMAT_MOD_INVALID); spa_pod_builder_pop(builder, &choice_frame); @@ -526,7 +604,9 @@ int osc_pw_enum_format_accepts_dmabuf_producer(int with_modifier, int64_t produc const struct spa_pod *consumer; const struct spa_pod *producer; - consumer = with_modifier ? osc_build_enum_format_dmabuf(&ours) : osc_build_enum_format(&ours); + /* The unit test exercises the full tiled offer, so enumerate unconditionally. */ + consumer = + with_modifier ? osc_build_enum_format_dmabuf(&ours, 1) : osc_build_enum_format(&ours); if (consumer == NULL) { return -1; } @@ -775,6 +855,52 @@ static void osc_dmabuf_sync(int fd, int start) } } +/* The live-buffer table is only touched on the PipeWire thread (add/remove_buffer) + * and, in osc_pw_requeue_buffer, under the thread-loop lock which pauses that + * thread — so these need no locking of their own. */ + +/* Registers `pw_buf` and returns the unique generation stamped on it, which the + * frame carries so a later re-queue can prove it means THIS registration and not + * a newer buffer reusing the same slot. 0 is never a valid generation. */ +static uint64_t osc_track_live_buffer(struct osc_pw_session *session, struct pw_buffer *pw_buf) +{ + size_t i; + uint64_t generation = ++session->next_generation; + for (i = 0; i < OSC_MAX_LIVE_BUFFERS; i++) { + if (session->live_buffers[i] == NULL) { + session->live_buffers[i] = pw_buf; + session->live_generations[i] = generation; + return generation; + } + } + return generation; +} + +static void osc_forget_live_buffer(struct osc_pw_session *session, struct pw_buffer *pw_buf) +{ + size_t i; + for (i = 0; i < OSC_MAX_LIVE_BUFFERS; i++) { + if (session->live_buffers[i] == pw_buf) { + session->live_buffers[i] = NULL; + session->live_generations[i] = 0; + return; + } + } +} + +/* The generation currently registered for `pw_buf`, or 0 if it is not tracked. */ +static uint64_t osc_live_buffer_generation(struct osc_pw_session *session, + struct pw_buffer *pw_buf) +{ + size_t i; + for (i = 0; i < OSC_MAX_LIVE_BUFFERS; i++) { + if (session->live_buffers[i] == pw_buf) { + return session->live_generations[i]; + } + } + return 0; +} + static void osc_on_add_buffer(void *userdata, struct pw_buffer *pw_buf) { struct osc_pw_session *session = userdata; @@ -786,6 +912,9 @@ static void osc_on_add_buffer(void *userdata, struct pw_buffer *pw_buf) if (pw_buf == NULL || pw_buf->buffer == NULL || pw_buf->buffer->n_datas < 1) { return; } + /* Record the buffer as live before anything else, so a handle the consumer + * holds can be validated against destruction in osc_pw_requeue_buffer. */ + osc_track_live_buffer(session, pw_buf); data = &pw_buf->buffer->datas[0]; if (data->type != SPA_DATA_DmaBuf) { return; @@ -805,23 +934,17 @@ static void osc_on_add_buffer(void *userdata, struct pw_buffer *pw_buf) maplen = data->maxsize; session->dmabuf_maps[i].ptr = osc_map_dmabuf((int)data->fd, &maplen, &why); if (session->dmabuf_maps[i].ptr == NULL) { - /* Reported once, through the buffer-info channel that already exists - * for describing what the compositor handed us — a mapping failure - * here means no frames at all, and silence would read as a hang. - * - * The reason is carried up rather than assumed: this used to say the - * driver refused CPU mapping no matter what actually went wrong, and - * that message sent the one real investigation of this path looking - * at the GPU for a size the compositor had simply left at 0. */ - if (session->callbacks.on_buffer_info != NULL && - session->buffer_info_reports < OSC_BUFFER_INFO_REPORTS) { - char detail[256]; - - snprintf(detail, sizeof(detail), "dmabuf import failed: %s; capture cannot proceed", - why); - session->buffer_info_reports++; - session->callbacks.on_buffer_info(session->callbacks.user, data->type, - pw_buf->buffer->n_datas, 0, 0, detail); + /* + * A mmap failure on a dmabuf is the tiled-buffer case (e.g. a whole + * monitor on AMD/mutter): the bytes are not in raster order and the + * driver refuses CPU access. That is no longer fatal — the frame + * instead travels as a raw dmabuf descriptor for a GPU import (see + * osc_read_frame and issue #507). Latch the mode; osc_read_frame will + * populate the descriptor from the same fd. No mapping is stored. + */ + session->import_dmabuf = 1; + if (osc_debug_enabled()) { + fprintf(stderr, "[osc-dmabuf] mmap failed (%s) — using GPU import path\n", why); } return; } @@ -840,6 +963,9 @@ static void osc_on_remove_buffer(void *userdata, struct pw_buffer *pw_buf) if (pw_buf == NULL || pw_buf->buffer == NULL || pw_buf->buffer->n_datas < 1) { return; } + /* The buffer is being destroyed: a consumer still holding it for a GPU import + * must not re-queue it. Forgetting it here makes osc_pw_requeue_buffer skip it. */ + osc_forget_live_buffer(session, pw_buf); data = &pw_buf->buffer->datas[0]; for (i = 0; i < OSC_MAX_DMABUF_MAPS; i++) { if (session->dmabuf_maps[i].ptr == NULL || @@ -985,6 +1111,7 @@ static int osc_read_frame(struct osc_pw_session *session, const struct spa_buffe uint32_t size; int32_t stride; int32_t height; + int is_dmabuf_import = 0; const uint8_t *base; @@ -1008,7 +1135,12 @@ static int osc_read_frame(struct osc_pw_session *session, const struct spa_buffe */ base = osc_find_dmabuf_map(session, (int)data->fd); if (base == NULL) { - return 0; + if (!session->import_dmabuf) { + return 0; + } + /* Tiled dmabuf: no CPU mapping exists. It travels up as a raw + * descriptor for a GPU import instead of being read here. */ + is_dmabuf_import = 1; } } else if (data->data == NULL) { /* @@ -1025,36 +1157,74 @@ static int osc_read_frame(struct osc_pw_session *session, const struct spa_buffe return 0; } - offset = SPA_MIN(data->chunk->offset, data->maxsize); - size = SPA_MIN(data->chunk->size, data->maxsize - offset); - height = (int32_t)session->format.size.height; stride = data->chunk->stride; - if (stride <= 0 || height <= 0) { - return 0; - } - /* One short row is one row of garbage in the recording; refuse the whole - * frame instead, and let the caller count it as dropped. */ - if ((uint64_t)stride * (uint64_t)height > (uint64_t)size) { + if (height <= 0) { return 0; } - /* - * Open the CPU-access window on a dmabuf and leave it open: the pixels are - * read by the on_frame callback, not here, so the matching SYNC_END lives in - * osc_inspect_buffer once that callback has returned. - */ - if (data->type == SPA_DATA_DmaBuf) { - session->dmabuf_sync_fd = (int)data->fd; - osc_dmabuf_sync(session->dmabuf_sync_fd, 1); - } + if (is_dmabuf_import) { + /* + * GPU import path. The buffer is not CPU-readable, so the raster bounds + * checks below do not apply — the modifier is what makes the producer's + * strides/offsets meaningful, and the importer validates the rest. We + * hand up the fd(s), modifier and fourcc; no SYNC bracket is opened + * because nothing here touches the pixels. n_datas is the plane count for + * a dmabuf (one per plane); our RGB formats are single-plane. + */ + uint32_t fourcc = osc_spa_format_to_drm_fourcc(session->format.format); + int32_t import_stride = + stride > 0 ? stride : (int32_t)session->format.size.width * 4; + int32_t p; + if (fourcc == 0) { + return 0; + } + out->is_dmabuf = 1; + out->data = NULL; + out->size = 0; + out->stride = import_stride; + out->width = (int32_t)session->format.size.width; + out->height = height; + out->video_format = session->format.format; + out->modifier = session->format.modifier; + out->drm_fourcc = fourcc; + out->n_planes = (int32_t)buffer->n_datas > 4 ? 4 : (int32_t)buffer->n_datas; + for (p = 0; p < out->n_planes; p++) { + const struct spa_data *pd = &buffer->datas[p]; + out->plane_fd[p] = (int)pd->fd; + out->plane_offset[p] = pd->chunk != NULL ? (int32_t)pd->chunk->offset : 0; + out->plane_stride[p] = + (pd->chunk != NULL && pd->chunk->stride > 0) ? pd->chunk->stride : import_stride; + } + } else { + offset = SPA_MIN(data->chunk->offset, data->maxsize); + size = SPA_MIN(data->chunk->size, data->maxsize - offset); + if (stride <= 0) { + return 0; + } + /* One short row is one row of garbage in the recording; refuse the whole + * frame instead, and let the caller count it as dropped. */ + if ((uint64_t)stride * (uint64_t)height > (uint64_t)size) { + return 0; + } - out->data = SPA_PTROFF(base, offset, const uint8_t); - out->size = size; - out->stride = stride; - out->width = (int32_t)session->format.size.width; - out->height = height; - out->video_format = session->format.format; + /* + * Open the CPU-access window on a dmabuf and leave it open: the pixels + * are read by the on_frame callback, not here, so the matching SYNC_END + * lives in osc_inspect_buffer once that callback has returned. + */ + if (data->type == SPA_DATA_DmaBuf) { + session->dmabuf_sync_fd = (int)data->fd; + osc_dmabuf_sync(session->dmabuf_sync_fd, 1); + } + + out->data = SPA_PTROFF(base, offset, const uint8_t); + out->size = size; + out->stride = stride; + out->width = (int32_t)session->format.size.width; + out->height = height; + out->video_format = session->format.format; + } header = spa_buffer_find_meta_data(buffer, SPA_META_Header, sizeof(*header)); if (header != NULL) { @@ -1152,8 +1322,11 @@ static void osc_describe_metas(const struct spa_buffer *buffer, char *out, size_ } } -static void osc_inspect_buffer(struct osc_pw_session *session, const struct spa_buffer *buffer) +/* Returns 1 when the on_frame callback took ownership of `pw_buf` (a dmabuf frame + * held for GPU import); the caller must then NOT re-queue it. 0 otherwise. */ +static int osc_inspect_buffer(struct osc_pw_session *session, struct pw_buffer *pw_buf) { + const struct spa_buffer *buffer = pw_buf->buffer; struct osc_pw_cursor cursor; uint32_t meta_size = 0; @@ -1185,7 +1358,17 @@ static void osc_inspect_buffer(struct osc_pw_session *session, const struct spa_ session->dmabuf_sync_fd = -1; if (osc_read_frame(session, buffer, &frame)) { - session->callbacks.on_frame(session->callbacks.user, &frame); + /* The callback needs the pw_buffer to hand back to osc_pw_requeue_buffer + * if it takes ownership of a dmabuf frame, plus its generation so the + * re-queue can reject a stale handle after a renegotiation. */ + frame.buffer_handle = pw_buf; + frame.buffer_generation = osc_live_buffer_generation(session, pw_buf); + if (session->callbacks.on_frame(session->callbacks.user, &frame)) { + /* Taken: leave it un-queued; the consumer will re-queue it once the + * import has copied the pixels. No SYNC bracket is open on this + * path (the import path does not CPU-read), so nothing to close. */ + return 1; + } } /* Closes the DMA_BUF_SYNC_START osc_read_frame opened, if any. Placed * here rather than inside it because the callback above is what actually @@ -1195,6 +1378,7 @@ static void osc_inspect_buffer(struct osc_pw_session *session, const struct spa_ session->dmabuf_sync_fd = -1; } } + return 0; } static void osc_on_process(void *userdata) @@ -1224,11 +1408,36 @@ static void osc_on_process(void *userdata) * throw away the cursor metadata riding on the same buffers. */ while ((b = api.stream_dequeue_buffer(session->stream)) != NULL) { - osc_inspect_buffer(session, b->buffer); - api.stream_queue_buffer(session->stream, b); + /* A dmabuf frame the consumer takes is held out of the queue until it has + * imported the pixels — see osc_pw_requeue_buffer. Everything else (shm, + * cursor-only buffers, declined frames) re-queues immediately. */ + if (!osc_inspect_buffer(session, b)) { + api.stream_queue_buffer(session->stream, b); + } } } +/* See the header. Locks the thread loop so a foreign thread can queue safely. */ +void osc_pw_requeue_buffer(struct osc_pw_session *session, void *buffer_handle, + uint64_t buffer_generation) +{ + if (session == NULL || buffer_handle == NULL || session->stream == NULL) { + return; + } + api.thread_loop_lock(session->loop); + /* Under the lock the PipeWire thread is paused, so the live-buffer table is + * stable. Re-queue only when the SAME registration is still live: matching + * the generation as well as the pointer rejects both a destroyed buffer and a + * newer one PipeWire placed in the same slot after a renegotiation. A stale + * handle is simply dropped — PipeWire already owns or freed that buffer. */ + if (buffer_generation != 0 && + osc_live_buffer_generation(session, (struct pw_buffer *)buffer_handle) == + buffer_generation) { + api.stream_queue_buffer(session->stream, (struct pw_buffer *)buffer_handle); + } + api.thread_loop_unlock(session->loop); +} + static const struct pw_stream_events osc_stream_events = { PW_VERSION_STREAM_EVENTS, .state_changed = osc_on_state_changed, @@ -1239,6 +1448,7 @@ static const struct pw_stream_events osc_stream_events = { }; struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video, + int prefer_dmabuf, const struct osc_pw_callbacks *callbacks, char *err, size_t err_len) { @@ -1264,6 +1474,7 @@ struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video, } session->callbacks = *callbacks; session->want_video = want_video; + session->prefer_dmabuf = prefer_dmabuf; /* calloc zeroes these, and 0 is a legitimate fd — so the "nothing pending" * sentinel has to be set explicitly. dmabuf_maps is keyed on ptr != NULL, * which calloc does get right. */ @@ -1319,21 +1530,23 @@ struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video, * previously failed the whole negotiation with "no more input formats". */ params[0] = osc_build_enum_format(&builder); - params[1] = osc_build_enum_format_dmabuf(&builder); + params[1] = osc_build_enum_format_dmabuf(&builder, session->prefer_dmabuf); /* - * Test affordance. Every compositor available for local testing — mutter, - * sway via xdg-desktop-portal-wlr — offers shm, so params[0] always wins and - * the DMA-BUF branch below (osc_map_dmabuf, the DMA_BUF_IOCTL_SYNC bracket, - * the dmabuf arm of osc_read_frame) never executes outside niri. Dropping - * the shm object leaves the producer no choice, which is the only way to - * exercise that code without the compositor from issue #287. + * When the GPU import path is available (prefer_dmabuf), offer dmabuf FIRST + * and shm SECOND: mutter then hands us a tiled dmabuf we import on the GPU + * (issue #507) instead of the shm buffer it throttles for a whole monitor. + * shm stays as the fallback, so a compositor that cannot produce dmabuf still + * negotiates on the shm object. The env var forces the same swap for testing + * on a machine where the probe would say no. * - * Never set in production: it would break exactly the compatibility the - * ordering above exists to preserve. + * Without either, the ordering is unchanged — shm first — so nothing moves on + * a build or driver without the VAAPI import. */ - if (getenv("OPENSCREEN_PIPEWIRE_FORCE_DMABUF") != NULL) { + if (session->prefer_dmabuf || getenv("OPENSCREEN_PIPEWIRE_FORCE_DMABUF") != NULL) { + const struct spa_pod *shm = params[0]; params[0] = params[1]; + params[1] = shm; } if (params[0] == NULL || params[1] == NULL) { diff --git a/electron/native/pipewire-capture/csrc/pw_shim.h b/electron/native/pipewire-capture/csrc/pw_shim.h index ab6f78305..23df69417 100644 --- a/electron/native/pipewire-capture/csrc/pw_shim.h +++ b/electron/native/pipewire-capture/csrc/pw_shim.h @@ -94,6 +94,37 @@ struct osc_pw_frame { * both draw the line in exactly this place. */ int has_crop; + + /* + * Zero-copy dmabuf hand-off (issue #507). When `is_dmabuf` is 1, `data` is + * NULL and the frame is not CPU-readable — a tiled compositor buffer that + * lives on the GPU. The consumer imports it as a VAAPI surface from the + * descriptor below instead of reading `data`. When 0, the CPU path above + * applies unchanged (shm, or a linear/implicit dmabuf we could mmap). + * + * When the on_frame callback TAKES a dmabuf frame (returns non-zero), the + * PipeWire buffer is NOT re-queued here — `buffer_handle` is retained by the + * consumer, which keeps the fds and their CONTENT valid until it has imported + * and copied the surface, then calls osc_pw_requeue_buffer. Duplicating the + * fds alone would preserve the dmabuf object but not a snapshot of its pixels, + * so a re-queued buffer the compositor overwrote could be encoded torn. + * `modifier`/`drm_fourcc` describe the tiling and pixel layout. + */ + int is_dmabuf; + uint64_t modifier; /* DRM format modifier of the buffer */ + uint32_t drm_fourcc; /* DRM fourcc matching `video_format` */ + int32_t n_planes; /* number of populated plane_* entries (1..4) */ + int plane_fd[4]; + int32_t plane_offset[4]; + int32_t plane_stride[4]; + /* The `struct pw_buffer *` this frame came from, opaque to the consumer. + * Passed back to osc_pw_requeue_buffer once the import is done. Only set (and + * only meaningful) for a dmabuf frame the consumer intends to take. */ + void *buffer_handle; + /* The registration generation of `buffer_handle`. Passed back alongside it so + * a re-queue can tell this buffer from a later one PipeWire put in the same + * slot after a renegotiation. */ + uint64_t buffer_generation; }; /* The negotiated video format. Reported once, from param_changed. */ @@ -113,8 +144,12 @@ struct osc_pw_callbacks { void *user; void (*on_format)(void *user, const struct osc_pw_format *format); void (*on_cursor)(void *user, const struct osc_pw_cursor *cursor); - /* Only ever called when osc_pw_start was given want_video != 0. */ - void (*on_frame)(void *user, const struct osc_pw_frame *frame); + /* Only ever called when osc_pw_start was given want_video != 0. Returns + * non-zero to TAKE OWNERSHIP of the PipeWire buffer (`frame->buffer_handle`): + * the shim then does NOT re-queue it, and the consumer must later call + * osc_pw_requeue_buffer. Zero (the shm/CPU path, and any dmabuf frame the + * consumer declines) re-queues immediately as before. */ + int (*on_frame)(void *user, const struct osc_pw_frame *frame); /* Emitted once per negotiated buffer set. `data_type` is the SPA_DATA_* of * datas[0]; `metas` is a borrowed "Header:12,Cursor:589872" listing of every * metadata block that survived negotiation, which is what distinguishes a @@ -197,12 +232,30 @@ const char *osc_pw_library_version(void); * buffer types; without it neither happens, and a cursor-only session never pays * to map a full-screen framebuffer per frame. * + * `prefer_dmabuf` offers dmabuf before shm so a tiled monitor buffer is imported + * on the GPU rather than copied through throttled shm (issue #507); set it only + * when the VAAPI import pipeline is available. shm remains the fallback. + * * Returns NULL on failure, with a message in `err`. */ struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video, + int prefer_dmabuf, const struct osc_pw_callbacks *callbacks, char *err, size_t err_len); +/* + * Re-queues a PipeWire buffer the on_frame callback took ownership of (returned + * non-zero for), identified by the `buffer_handle` it was given. Call it once the + * frame's pixels have been imported and copied. + * + * SAFE TO CALL FROM ANY THREAD: it takes the PipeWire thread-loop lock around the + * queue, so unlike the shim's own callbacks it must NOT be called from the + * PipeWire thread itself (that would deadlock). The consumer requeues from its + * own loop, which is a different thread. NULL session or handle is a no-op. + */ +void osc_pw_requeue_buffer(struct osc_pw_session *session, void *buffer_handle, + uint64_t buffer_generation); + /* Stops the thread loop, joins it, and frees everything. Safe with NULL. */ void osc_pw_stop(struct osc_pw_session *session); diff --git a/electron/native/pipewire-capture/docs/dmabuf-vaapi-plan.md b/electron/native/pipewire-capture/docs/dmabuf-vaapi-plan.md new file mode 100644 index 000000000..aa25576e4 --- /dev/null +++ b/electron/native/pipewire-capture/docs/dmabuf-vaapi-plan.md @@ -0,0 +1,167 @@ +# Zero-copy dmabuf → VAAPI capture (fix for #507) + +## Problem + +On GNOME/mutter + AMD, the helper only advertises `LINEAR`/`INVALID` dmabuf +modifiers (it reads frames via CPU `mmap`, which needs linear). AMD monitor +buffers are **tiled**, so dmabuf negotiation can't succeed and we fall back to +**shm/memfd**. mutter throttles whole-monitor shm delivery hard (GPU→CPU copy +per frame), starving the recorder to ~2–11 distinct fps while OBS gets ~24 over +dmabuf. Result: whole-screen recordings look frozen. Window capture is less +affected (smaller surface → shm copy keeps up). + +Goal: import the compositor's **tiled** dmabuf directly as a VAAPI surface and +encode with the existing `h264_vaapi` path — no CPU readback, no shm. + +## Constraint that shapes the design: the clock-driven encoder + +`capture.rs` writes constant-frame-rate output by *holding the last staged +picture* across gaps (a static screen delivers no frames). We therefore cannot +pin the PipeWire dmabuf across that gap — the pool is 4–16 buffers. So on each +arriving frame we must copy it into a surface **we own**, then requeue the +compositor buffer promptly. The copy stays on the GPU (VAAPI VPP), so it's cheap. + +## Pipeline (new dmabuf path, shm path kept as fallback) + +1. **Negotiation** — advertise the dmabuf modifiers our importer supports. + Enumerate them from the DRM render node (VAAPI/`vaQuerySurfaceAttributes` or + EGL `eglQueryDmaBufModifiersEXT`), per fourcc, like OBS. Offer that list in + `osc_build_enum_format_dmabuf` instead of just LINEAR/INVALID. +2. **C shim** — on `SPA_DATA_DmaBuf`, stop mmap'ing. Extract the raw descriptor: + fd(s), `format_modifier`, and per-plane `offset`/`stride`, plus fourcc. Pass + them to Rust via an extended `osc_pw_frame`/`RawFrame`. Keep the + `DMA_BUF_IOCTL_SYNC` bracket only for the (unused-on-dmabuf) CPU path. +3. **Frame lifecycle** — the mailbox must not `memcpy` for dmabuf. It holds the + descriptor + a handle that keeps the PipeWire buffer un-requeued until the + main loop imports it; newest-wins requeues the superseded buffer. Requeue + happens right after import (fast), never across the clock gap. +4. **Encoder** — build an `AVFrame` of `AV_PIX_FMT_DRM_PRIME` wrapping an + `AVDRMFrameDescriptor`, `av_hwframe_map()` it to a VAAPI frame (DRM→VAAPI + zero-copy), then VPP (`scale_vaapi`/`vpp_vaapi`) into our own NV12 VAAPI pool + surface — this also applies the **crop** (VideoCrop) on the GPU, replacing the + current CPU pointer-offset crop. That owned surface becomes the staged frame; + `encode_staged` sends it directly (no `av_hwframe_transfer_data` upload). +5. **Fallback** — the dmabuf path is NOT GNOME-specific: it applies to any + compositor that offers dmabuf (GNOME/mutter, KDE/kwin, most wlroots) whenever + the encoder is **VAAPI** (the default Linux backend with any GPU), so the large + majority of PipeWire desktop users benefit. shm stays as the fallback only for: + (a) compositors that offer *only* shm (some `xdg-desktop-portal-wlr` configs — + why shm is listed first today), and (b) non-VAAPI encoders (software + libopenh264 / Vulkan), whose dmabuf import isn't wired yet — they keep today's + shm + sws_scale + hwupload path. Also fall back if enumeration/import/VPP fails. + Never regress software-encode or shm-only-compositor users. + +## Decision: zero-copy (option B) + +Chosen over the pragmatic GPU-detile→CPU-readback path. The dmabuf stays on the +GPU end to end: `av_hwframe_map` (DRM_PRIME→VAAPI) → `scale_vaapi` VPP (format + +crop) into our own NV12 surface → encode. No CPU readback. + +Key architecture calls: +- **One shared VAAPI `AVHWDeviceContext`** created up front, used by BOTH the + importer (PipeWire thread) and the encoder (main loop). A single mutex guards + all VADisplay ops (import+VPP vs encode) since libva isn't thread-safe per + display. Contention is negligible (both are GPU-driven). +- **Import runs on the PipeWire thread inside `on_frame`**, while the PW buffer is + still held (before requeue), so the dmabuf content is stable during the map+VPP + copy. The result is our own NV12 VAAPI surface (ref-counted `AVFrame`) placed in + the mailbox; the PW buffer requeues immediately after. This preserves the + clock-driven hold (we own the surface; the compositor buffer is returned). +- **v1 targets full monitor (no crop)**: importer/VPP sized to the stream at + `stream-started`. Window crop via VPP is a follow-up. +- **Fallback** to the existing shm + sws_scale + hwupload path when: backend isn't + VAAPI, the compositor only offers shm, or any of map/VPP/import fails. + +## Status / steps + +- [x] **Foundation**: generate ffmpeg DRM bindings (`build.rs` + + `hwcontext_drm.h`). Verified `AVDRMFrameDescriptor`, `AV_PIX_FMT_DRM_PRIME`, + `AV_HWDEVICE_TYPE_DRM` present. +- [x] **Negotiation** (validated on AMD/mutter): enumerate importable modifiers + via EGL surfaceless (`csrc/dmabuf_modifiers.c`) and advertise them in + `osc_build_enum_format_dmabuf`. Confirmed mutter now negotiates a **tiled + dmabuf** (`stream-started` fires) where before it failed with "no more input + formats". Enumeration returns 10 AMD GFX9 modifiers for XRGB8888. The existing + `mmap` path then correctly reports "driver does not allow CPU mapping" — the + exact branch point for the GPU import below. +- [x] Extend `osc_pw_frame` (pw_shim.h) + `RawFrame` (shim.rs) with + is_dmabuf/modifier/fourcc/n_planes/plane_fd/offset/stride. Layouts mirror + exactly; builds green. +- [x] C: `osc_read_frame` populates the descriptor for a tiled dmabuf; + `osc_on_add_buffer` latches `import_dmabuf` on mmap failure instead of erroring; + fourcc mapping added. shm/linear-dmabuf paths unchanged. `on_frame` currently + skips dmabuf frames (data==null) — safe no-op until the importer lands. +- [x] Build foundation for the importer: vendored **libavfilter** wired in + (bindgen headers `avfilter.h`/`buffersrc.h`/`buffersink.h` + allowlist, link, + and staged into `helper-ffmpeg/` by the build script). `av_hwframe_map`, + `av_hwdevice_ctx_create_derived`, `AVDRMFrameDescriptor`, `AV_PIX_FMT_DRM_PRIME`, + `avfilter_graph_*`, `av_buffersrc/sink_*` all generate and link. Builds green. + +### Importer design decisions (settled while scoping) + +- **v1 buffer lifetime**: `on_frame` (PW thread) `dup()`s the plane fds into + `OwnedFd`s (std, no libc), puts the descriptor in the mailbox, and requeues the + PW buffer normally. The map+VPP+encode runs on the **main loop** — all VAAPI on + one thread, no cross-thread device or mutex. The fds keep the dmabuf alive for + the import; content-tear risk (compositor reusing the requeued buffer before the + main loop imports, ~1 tick later) is low with a 4–16 buffer pool and is the one + thing to watch. Upgrade to buffer-holding only if tearing shows. +- **Device & pool ownership**: create ONE standalone VAAPI `AVHWDeviceContext`; + derive a DRM device from it for the DRM_PRIME source frames ctx. Build the + `scale_vaapi` filtergraph (buffersrc VAAPI-BGR0 → `format=nv12` → buffersink) + and take the buffersink's **output NV12 hw_frames_ctx** as the encoder's + `codec_ctx->hw_frames_ctx`. That means for the dmabuf path the **encoder is + opened AFTER the importer/filtergraph is built**, so their pools match and + `avcodec_send_frame` accepts the surface directly. +- **Per frame**: build `AVDRMFrameDescriptor` (1 object: fd/size/modifier; 1 + layer: fourcc; 1 plane: offset/pitch) → DRM_PRIME AVFrame → `av_hwframe_map` + DIRECT|READ → VAAPI BGR0 → buffersrc→scale_vaapi→buffersink → NV12 VAAPI → + `encoder.stage_hw()` (held as `hw_staged`; `encode_staged` sends it with pts, + no unref between the clock-driven re-encodes). + +### Remaining +- [x] `dmabuf_import.rs`: the map + scale_vaapi VPP → NV12 module. +- [x] `shim.rs`: `DmabufDesc` (OwnedFd planes) + `Frame.dmabuf` + `on_frame` dup + + mailbox `put_dmabuf` (no memcpy). +- [x] `encoder.rs`: `open_importing` (shared device + external NV12 pool) + + `stage_hw`/`hw_staged` path (held across re-encodes) + Drop. +- [x] `capture.rs`: dmabuf branch → importer → `stage_hw`; deferred encoder open. +- [x] Whole pipeline compiles and links; full helper builds, libavfilter staged. +- [x] **On-device validated** (AMD/radeonsi + mutter, via `FORCE_DMABUF`). + Full-monitor editor scroll: **42.4 distinct fps** (was ~2 on shm; OBS ~24), + `convertMs 0.0`, `uploadMs ~0.002` — the frame never touches the CPU. Four + runtime fixes were needed and are in: (1) create the DRM device on the render + node and derive VAAPI from it — the reverse is ENOSYS on radeonsi; (2) + `initial_pool_size = 0` on the map-only frames contexts; (3) allocate the + buffersrc then set params (hw_frames_ctx) then init, since a HW pix_fmt is + rejected at init otherwise; (4) wrap the DRM descriptor in an AVBufferRef so the + source frame is ref-counted for `av_hwframe_map`. +- [x] **Auto-enable + fallback** (validated, no env). `dmabuf_import::available()` + probes once at session start by building a nominal importer; when it succeeds + the stream offers dmabuf BEFORE shm (`osc_pw_start(prefer_dmabuf)`), else stays + on shm. shm remains in the offer as the negotiation fallback, so a compositor + that cannot produce dmabuf — or a GPU where the importer will not build — keeps + today's path with no regression. Confirmed: full-monitor scroll records 39.8 + distinct fps with `convertMs 0.0` and no force flag. `OPENSCREEN_PIPEWIRE_FORCE_DMABUF` + still forces the swap for testing. +- [x] **Window crop via VPP** (validated). The importer now takes a source size + (the full stream) and an output size (the committed crop); `scale_vaapi` outputs + the crop size and `import` sets the mapped surface's crop_left/top/right/bottom + per frame so the VA source region is the window rect — cropped and format- + converted on the GPU, no scaling (region == output). Confirmed: a 724×576 GNOME + window records at 724×576, sharp, convertMs 0.0, 26.6 distinct fps. (The black + margin some CSD windows show is the shadow/decoration in mutter's crop rect — + same on any capture tool, not introduced here.) +- [ ] Follow-ups: per-frame import failure after a successful probe still errors + (rare) rather than renegotiating to shm; test on Intel/NVIDIA-vaapi. +- [ ] Test on AMD/GNOME: confirm `uses_dmabuf=1`, distinct-fps ≈ OBS (~24), + crop correct for window captures, cursor unaffected. Regression-check + software encode and a wlroots/niri compositor. + +## Risk notes + +- radeonsi VAAPI must import the specific tiled modifier mutter exports — highly + likely OK (GNOME/OBS do DRM→VAAPI on this GPU), but the concrete failure mode + is `av_hwframe_map` returning an error → must fall back cleanly. +- Concurrency: import/VPP needs the VAAPI context; keep it on the main loop + (as sws_scale is today), holding the PW buffer only until the next tick. diff --git a/electron/native/pipewire-capture/src/capture.rs b/electron/native/pipewire-capture/src/capture.rs index 17c42f801..38452b618 100644 --- a/electron/native/pipewire-capture/src/capture.rs +++ b/electron/native/pipewire-capture/src/capture.rs @@ -192,6 +192,11 @@ pub struct Summary { pub struct Capture { encoder: VideoEncoder, + /// The dmabuf → VAAPI importer, present only on the zero-copy path (issue + /// #507). When set, [`Self::stage`] imports the frame's descriptor into an + /// NV12 surface instead of running swscale, and the encoder was opened + /// against this importer's pool. + importer: Option, video_track: TrackId, audio: Option, /// `None` only between [`Self::finish`] taking it and the struct dropping. @@ -222,6 +227,18 @@ pub struct Capture { committed_height: i32, } +/// The outcome of staging one captured frame. +#[derive(Debug, PartialEq, Eq)] +pub enum StageOutcome { + /// A new frame was staged; `advance` will encode it. + Staged, + /// A recoverable per-frame failure — one dmabuf the GPU could not map, or a + /// transient EAGAIN. The frame is skipped and `advance` holds the previously + /// staged one forward, so a single bad frame costs one frame, not the whole + /// recording. Carries the reason for a log warning. + Dropped(String), +} + impl Capture { pub fn start( path: &Path, @@ -233,14 +250,51 @@ impl Capture { bitrate: Option, forced: Option, audio_sources: Vec, + // Present when the first frame is a tiled dmabuf: the encoder is then + // opened to consume the importer's NV12 pool directly (issue #507). + dmabuf: Option<&shim::DmabufDesc>, ) -> Result<(Self, Selection), String> { let bitrate = bitrate.unwrap_or_else(|| default_bitrate(width, height, fps)); let mut rejected = Vec::new(); - let encoder = VideoEncoder::open( - VideoParams { width, height, fps, bitrate }, - forced, - |backend, error| rejected.push(format!("{}: {error}", backend.as_str())), - )?; + // Shared with the negotiation offer (`prefer_dmabuf` in main) so the two + // never disagree: offering dmabuf that this then refuses to import is what + // makes a forced-software recording fail on its first frame. + let use_dmabuf = crate::encoder::forced_allows_dmabuf(forced); + let (encoder, importer) = match dmabuf.filter(|_| use_dmabuf) { + Some(desc) => { + // The importer maps the full stream (`desc`) and its VPP crops to + // the committed record size (`width`/`height`): equal to the source + // for a monitor, or the window's crop rectangle for a window. The + // encoder is FORCED to VAAPI — the only backend that can consume the + // mapped surface; a non-VAAPI machine never negotiates dmabuf. + let importer = crate::dmabuf_import::DmabufImporter::new( + desc.width, + desc.height, + width, + height, + desc.drm_fourcc, + )?; + // SAFETY: the importer's device and NV12 frames context are live + // for as long as the returned encoder, which the Capture owns + // alongside it below. + let encoder = unsafe { + VideoEncoder::open_importing( + VideoParams { width, height, fps, bitrate }, + importer.device(), + importer.output_frames_ctx(), + )? + }; + (encoder, Some(importer)) + } + None => { + let encoder = VideoEncoder::open( + VideoParams { width, height, fps, bitrate }, + forced, + |backend, error| rejected.push(format!("{}: {error}", backend.as_str())), + )?; + (encoder, None) + } + }; let selection = Selection { backend: encoder.backend(), rejected }; // Every track must exist before the header: MP4 fixes its track list @@ -274,6 +328,7 @@ impl Capture { Ok(( Self { encoder, + importer, video_track, audio, muxer: Some(muxer), @@ -321,8 +376,50 @@ impl Capture { } /// Converts a captured frame into the encoder's staging buffer. Nothing is - /// written until [`Self::advance`] runs. - pub fn stage(&mut self, frame: &shim::Frame) -> Result<(), String> { + /// written until [`Self::advance`] runs. A recoverable per-frame failure (a + /// dmabuf the GPU cannot map) returns `Ok(Dropped)` rather than `Err`, so it + /// costs one frame, not the recording; a genuine encoder error still errors. + pub fn stage(&mut self, frame: &shim::Frame) -> Result { + // Zero-copy dmabuf path: import the tiled GPU buffer into an NV12 VAAPI + // surface (the VPP crops a window to its committed rectangle) and hand it + // to the encoder as-is — no swscale. See issue #507. + if frame.dmabuf.is_some() { + // The crop origin, clamped to stay inside the buffer — same rule as the + // CPU path. Computed before the mutable importer borrow. For a monitor + // this is (0, 0). + let (crop_x, crop_y) = self.read_origin(frame); + let desc = frame.dmabuf.as_ref().expect("checked is_some above"); + let importer = self + .importer + .as_mut() + .ok_or_else(|| "dmabuf frame arrived but no importer was built".to_owned())?; + // Borrow the descriptor's planes directly — they are already + // `shim::DmabufPlane`, the exact type `import` takes — instead of + // reallocating a plane vector on every frame. + let nv12 = match importer.import( + &crate::dmabuf_import::DmabufFrame { + width: desc.width, + height: desc.height, + drm_fourcc: desc.drm_fourcc, + modifier: desc.modifier, + planes: &desc.planes, + }, + crop_x, + crop_y, + ) { + Ok(nv12) => nv12, + // A single un-mappable buffer must not end the recording. Skip it; + // `advance` holds the previous frame, and the shm path is still in + // the offer for a full downgrade later (a planned follow-up). + Err(reason) => return Ok(StageOutcome::Dropped(reason)), + }; + // SAFETY: `nv12` is a VAAPI NV12 frame from the pool the encoder was + // opened against; the encoder takes ownership. + unsafe { self.encoder.stage_hw(nv12) }; + self.mark_started(); + return Ok(StageOutcome::Staged); + } + let format = pixel_format(frame.video_format)?; // Address the crop by moving the START of the slice, and hand swscale the @@ -341,21 +438,28 @@ impl Capture { .ok_or_else(|| format!("crop offset {offset} is past the end of the frame"))?; self.encoder.stage(pixels, frame.stride, format)?; - if self.epoch.is_none() { - self.epoch = Some(Instant::now()); - // Audio has been accumulating since the process started, while the - // portal picker was up and the format was being negotiated. None of - // it belongs to the recording: video frame 0 is now, so audio - // sample 0 is now too. Keeping the backlog would shift the whole - // track earlier by however long the user took to click. - if let Some(mix) = &mut self.audio { - for input in &mut mix.inputs { - input.ring.clear(); - input.pending.clear(); - } + self.mark_started(); + Ok(StageOutcome::Staged) + } + + /// Starts the timeline on the first staged frame and drops the audio backlog. + /// + /// Audio has been accumulating since the process started, while the portal + /// picker was up and the format was being negotiated. None of it belongs to + /// the recording: video frame 0 is now, so audio sample 0 is now too. Keeping + /// the backlog would shift the whole track earlier by however long the user + /// took to click. + fn mark_started(&mut self) { + if self.epoch.is_some() { + return; + } + self.epoch = Some(Instant::now()); + if let Some(mix) = &mut self.audio { + for input in &mut mix.inputs { + input.ring.clear(); + input.pending.clear(); } } - Ok(()) } /// Whether a picture has been staged, which is also whether the timeline has @@ -546,6 +650,7 @@ mod tests { pts_ns: -1, crop: shim::CropRect { x: 0, y: 0, width, height }, has_crop: false, + dmabuf: None, } } @@ -590,7 +695,7 @@ mod tests { fn the_timeline_does_not_start_until_the_first_frame_is_staged() { let output = std::env::temp_dir().join("openscreen-capture-epoch.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); assert!(!capture.started()); // Nothing staged: advance must not write a frame of uninitialised memory. @@ -609,7 +714,7 @@ mod tests { // further arrivals, and the file must still fill with frames. let output = std::env::temp_dir().join("openscreen-capture-static.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); capture .stage(&frame(320, 240, shim::constants().video_format_bgrx)) @@ -634,7 +739,7 @@ mod tests { fn a_window_is_staged_from_its_crop_inside_a_larger_frame() { let output = std::env::temp_dir().join("openscreen-capture-crop.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); // A 1920x1080 stream carrying a 320x240 window at (100, 50). @@ -665,7 +770,7 @@ mod tests { fn a_crop_against_the_right_edge_is_not_rejected_as_truncated() { let output = std::env::temp_dir().join("openscreen-capture-edge.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); let staged = capture.stage(&cropped_frame( @@ -688,7 +793,7 @@ mod tests { fn a_shrunken_window_is_read_from_inside_the_frame() { let output = std::env::temp_dir().join("openscreen-capture-shrunk.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); // Origin so close to the edge that a 320x240 read from it would overrun. @@ -714,7 +819,7 @@ mod tests { let output = std::env::temp_dir().join("openscreen-capture-odd.mp4"); // 321x241 rounds to the 320x240 the encoder is opened at. let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); let frame = cropped_frame( @@ -733,7 +838,7 @@ mod tests { fn an_uncropped_frame_reports_no_divergence() { let output = std::env::temp_dir().join("openscreen-capture-nocrop.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); assert!(!capture.crop_diverged(&frame(320, 240, shim::constants().video_format_bgrx))); @@ -745,7 +850,7 @@ mod tests { fn paused_time_does_not_advance_the_timeline() { let output = std::env::temp_dir().join("openscreen-capture-pause.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 30, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); capture .stage(&frame(320, 240, shim::constants().video_format_bgrx)) @@ -785,6 +890,7 @@ mod tests { Some(1_000_000), Some(Backend::Software), vec![AudioSource { label: "system", ring: ring.clone(), gain: 1.0, bitrate: 128_000 }], + None, ) .expect("start"); @@ -821,6 +927,7 @@ mod tests { Some(1_000_000), Some(Backend::Software), vec![AudioSource { label: "system", ring: ring.clone(), gain: 1.0, bitrate: 128_000 }], + None, ) .expect("start"); capture @@ -863,6 +970,7 @@ mod tests { Some(1_000_000), Some(Backend::Software), vec![AudioSource { label: "microphone", ring, gain: 4.0, bitrate: 128_000 }], + None, ) .expect("start"); capture @@ -902,6 +1010,7 @@ mod tests { AudioSource { label: "system", ring: system.clone(), gain: 1.0, bitrate: 128_000 }, AudioSource { label: "microphone", ring: mic.clone(), gain: 1.0, bitrate: 128_000 }, ], + None, ) .expect("start"); @@ -951,6 +1060,7 @@ mod tests { AudioSource { label: "system", ring: system.clone(), gain: 1.0, bitrate: 128_000 }, AudioSource { label: "microphone", ring: dead, gain: 1.0, bitrate: 128_000 }, ], + None, ) .expect("start"); capture @@ -974,7 +1084,7 @@ mod tests { fn catch_up_is_bounded_so_a_stall_cannot_block_stop() { let output = std::env::temp_dir().join("openscreen-capture-catchup.mp4"); let (mut capture, _) = - Capture::start(&output, 320, 240, 60, Some(1_000_000), Some(Backend::Software), Vec::new()) + Capture::start(&output, 320, 240, 60, Some(1_000_000), Some(Backend::Software), Vec::new(), None) .expect("start"); capture .stage(&frame(320, 240, shim::constants().video_format_bgrx)) diff --git a/electron/native/pipewire-capture/src/dmabuf_import.rs b/electron/native/pipewire-capture/src/dmabuf_import.rs new file mode 100644 index 000000000..c3dc23553 --- /dev/null +++ b/electron/native/pipewire-capture/src/dmabuf_import.rs @@ -0,0 +1,486 @@ +//! Zero-copy import of a compositor dmabuf into a VAAPI NV12 surface (issue #507). +//! +//! On GNOME/Wayland + AMD a whole-monitor capture arrives as a *tiled* dmabuf +//! that cannot be CPU-mapped. Rather than fall back to the shm path (which mutter +//! throttles hard), we keep the frame on the GPU: wrap the dmabuf as a +//! `DRM_PRIME` frame, [`av_hwframe_map`] it into a VAAPI surface, and run it +//! through a `scale_vaapi` VPP that converts the BGRx layout to the NV12 the +//! H.264 VAAPI encoder wants. Nothing is read back to system memory. +//! +//! The importer owns the VAAPI device and the filtergraph. The encoder is opened +//! against [`Self::output_frames_ctx`] so the NV12 surface this produces is one +//! `avcodec_send_frame` accepts directly. See docs/dmabuf-vaapi-plan.md. + +use crate::ffmpeg as ff; +use std::os::fd::AsRawFd; +use std::ptr; + +/// `av_frame_free` wants a `**AVFrame`; wrap the pointer in a local so the null +/// it writes back does not land in a temporary. +unsafe fn free_frame(frame: *mut ff::AVFrame) { + let mut p = frame; + ff::av_frame_free(&mut p); +} + +/// Frees the heap-allocated `AVDRMFrameDescriptor` when its AVBufferRef drops — +/// which is after the mapped frame that retained the source (and thus this +/// buffer) is released. +unsafe extern "C" fn drm_descriptor_free(_opaque: *mut std::ffi::c_void, data: *mut u8) { + ff::av_free(data as *mut std::ffi::c_void); +} + +/// A dmabuf frame to import. Reuses [`crate::shim::DmabufPlane`] (an identical +/// `{fd, offset, stride}`) rather than a second copy, so the capture path can +/// borrow `DmabufDesc::planes` straight through instead of reallocating a plane +/// vector per frame — on the one path whose whole purpose is avoiding per-frame +/// copies. The fds are borrowed for the duration of [`DmabufImporter::import`] +/// only: VAAPI dups them during surface creation, so the caller may close after. +pub struct DmabufFrame<'a> { + pub width: i32, + pub height: i32, + pub drm_fourcc: u32, + pub modifier: u64, + pub planes: &'a [crate::shim::DmabufPlane], +} + +/// The pixel format the VAAPI-mapped surface presents, derived from the dmabuf's +/// DRM fourcc. Only the 32-bit RGB layouts we negotiate are handled. +fn sw_format_for_fourcc(drm_fourcc: u32) -> Option { + // DRM fourccs (little-endian) → the matching packed ffmpeg format. + const XRGB8888: u32 = 0x34325258; // SPA BGRx + const ARGB8888: u32 = 0x34325241; // SPA BGRA + const XBGR8888: u32 = 0x34324258; // SPA RGBx + const ABGR8888: u32 = 0x34324241; // SPA RGBA + match drm_fourcc { + XRGB8888 => Some(ff::AV_PIX_FMT_BGR0), + ARGB8888 => Some(ff::AV_PIX_FMT_BGRA), + XBGR8888 => Some(ff::AV_PIX_FMT_0BGR), + ABGR8888 => Some(ff::AV_PIX_FMT_ABGR), + _ => None, + } +} + +/// Whether the zero-copy VAAPI dmabuf-import pipeline can be built on this +/// machine. Constructs a nominal importer, which exercises the DRM→VAAPI device +/// creation, the frames contexts and the `scale_vaapi` graph — everything that +/// fails on a non-VAAPI GPU or a driver that cannot map a dmabuf. Success does +/// not depend on the exact dimensions, so a fixed probe size is representative. +/// When this is true the stream prefers dmabuf; when false it stays on shm. +pub fn available() -> bool { + const XRGB8888: u32 = 0x34325258; + DmabufImporter::new(1920, 1080, 1920, 1080, XRGB8888).is_ok() +} + +pub struct DmabufImporter { + /// Size of the incoming dmabuf (the whole stream). For a window this is the + /// monitor; for a monitor it equals the output size. + src_width: i32, + src_height: i32, + /// Size of the NV12 the graph emits — the recorded size. For a window this is + /// the committed crop rectangle; for a monitor it equals the source size. + out_width: i32, + out_height: i32, + sw_format: ff::AVPixelFormat, + /// VAAPI device, shared with the encoder (whose `hw_frames_ctx` comes from + /// [`Self::output_frames_ctx`]). + va_device: *mut ff::AVBufferRef, + /// DRM device derived from `va_device`; backs the DRM_PRIME source frames. + drm_device: *mut ff::AVBufferRef, + /// Frames context for the incoming DRM_PRIME buffers. + drm_frames: *mut ff::AVBufferRef, + /// Frames context for the VAAPI surface the dmabuf maps into (still BGRx). + va_map_frames: *mut ff::AVBufferRef, + graph: *mut ff::AVFilterGraph, + buffersrc_ctx: *mut ff::AVFilterContext, + buffersink_ctx: *mut ff::AVFilterContext, +} + +impl DmabufImporter { + /// Builds the device, frames contexts and `scale_vaapi` graph. `src` is the + /// incoming dmabuf size (the whole stream); `out` is the recorded size — equal + /// to `src` for a monitor, or the window's crop rectangle for a window (the + /// graph then crops the source region down to it, on the GPU). + pub fn new( + src_width: i32, + src_height: i32, + out_width: i32, + out_height: i32, + drm_fourcc: u32, + ) -> Result { + let sw_format = + sw_format_for_fourcc(drm_fourcc).ok_or_else(|| format!("unsupported dmabuf fourcc {drm_fourcc:#x}"))?; + + // SAFETY: every pointer is checked before use and freed in Drop. + unsafe { + let mut me = DmabufImporter { + src_width, + src_height, + out_width, + out_height, + sw_format, + va_device: ptr::null_mut(), + drm_device: ptr::null_mut(), + drm_frames: ptr::null_mut(), + va_map_frames: ptr::null_mut(), + graph: ptr::null_mut(), + buffersrc_ctx: ptr::null_mut(), + buffersink_ctx: ptr::null_mut(), + }; + + // Order matters: create the DRM device on the render node FIRST, then + // derive VAAPI from it. The reverse (DRM derived from VAAPI) returns + // ENOSYS on radeonsi — VAAPI knows how to open on a DRM fd, but not the + // other way round. The DRM device backs the DRM_PRIME source frames; + // the derived VAAPI device backs the mapped surface and the encoder. + // /dev/dri/renderD128 is the default, but on hybrid graphics the + // compositor may render on a different node — mapping a dmabuf from the + // wrong GPU then fails on every frame, with `available()` still passing. + // Honour an override so such a machine can point at the right node + // without a rebuild; the single-GPU default is unchanged. + let node_path = std::env::var("OPENSCREEN_LINUX_RENDER_NODE") + .unwrap_or_else(|_| "/dev/dri/renderD128".to_owned()); + let node = std::ffi::CString::new(node_path) + .map_err(|_| "OPENSCREEN_LINUX_RENDER_NODE has an interior NUL byte".to_owned())?; + let created = ff::av_hwdevice_ctx_create( + &mut me.drm_device, + ff::AV_HWDEVICE_TYPE_DRM, + node.as_ptr(), + ptr::null_mut(), + 0, + ); + if created < 0 { + return Err(format!("av_hwdevice_ctx_create(DRM): {}", ff::err_to_string(created))); + } + + let derived = ff::av_hwdevice_ctx_create_derived( + &mut me.va_device, + ff::AV_HWDEVICE_TYPE_VAAPI, + me.drm_device, + 0, + ); + if derived < 0 { + return Err(format!( + "av_hwdevice_ctx_create_derived(VAAPI): {}", + ff::err_to_string(derived) + )); + } + + me.drm_frames = me.alloc_frames(me.drm_device, ff::AV_PIX_FMT_DRM_PRIME)?; + me.va_map_frames = me.alloc_frames(me.va_device, ff::AV_PIX_FMT_VAAPI)?; + me.build_graph()?; + Ok(me) + } + } + + /// Allocates and initialises a frames context of `hw_format` (VAAPI or + /// DRM_PRIME) whose software format is the stream's RGB layout. + unsafe fn alloc_frames( + &self, + device: *mut ff::AVBufferRef, + hw_format: ff::AVPixelFormat, + ) -> Result<*mut ff::AVBufferRef, String> { + let frames = ff::av_hwframe_ctx_alloc(device); + if frames.is_null() { + return Err("av_hwframe_ctx_alloc failed".to_owned()); + } + let ctx = (*frames).data as *mut ff::AVHWFramesContext; + (*ctx).format = hw_format; + (*ctx).sw_format = self.sw_format; + (*ctx).width = self.src_width; + (*ctx).height = self.src_height; + // Pool size 0: these contexts only WRAP/MAP externally-supplied surfaces + // (the DRM_PRIME source is our imported dmabuf; the VAAPI context is filled + // by av_hwframe_map DIRECT). Asking for a pre-allocated pool makes + // av_hwframe_ctx_init reject the format with EINVAL, since neither has an + // allocator for these RGB layouts. + (*ctx).initial_pool_size = 0; + let init = ff::av_hwframe_ctx_init(frames); + if init < 0 { + let mut f = frames; + ff::av_buffer_unref(&mut f); + return Err(format!("av_hwframe_ctx_init: {}", ff::err_to_string(init))); + } + Ok(frames) + } + + /// Builds `buffer (VAAPI/BGRx) -> scale_vaapi=format=nv12 -> buffersink`. + unsafe fn build_graph(&mut self) -> Result<(), String> { + self.graph = ff::avfilter_graph_alloc(); + if self.graph.is_null() { + return Err("avfilter_graph_alloc failed".to_owned()); + } + + let buffersrc = ff::avfilter_get_by_name(c"buffer".as_ptr()); + let buffersink = ff::avfilter_get_by_name(c"buffersink".as_ptr()); + let scale = ff::avfilter_get_by_name(c"scale_vaapi".as_ptr()); + if buffersrc.is_null() || buffersink.is_null() || scale.is_null() { + return Err("a required filter (buffer/buffersink/scale_vaapi) is missing".to_owned()); + } + + // buffersrc: the input is a VAAPI surface. Allocate WITHOUT initialising + // (avfilter_graph_alloc_filter, not ..._create_filter): a hardware pix_fmt + // is rejected at init unless hw_frames_ctx is already set, and only + // av_buffersrc_parameters_set can set it. So: alloc → set params → init. + self.buffersrc_ctx = ff::avfilter_graph_alloc_filter(self.graph, buffersrc, c"in".as_ptr()); + if self.buffersrc_ctx.is_null() { + return Err("avfilter_graph_alloc_filter(buffersrc) failed".to_owned()); + } + let par = ff::av_buffersrc_parameters_alloc(); + if par.is_null() { + return Err("av_buffersrc_parameters_alloc failed".to_owned()); + } + (*par).format = ff::AV_PIX_FMT_VAAPI as i32; + (*par).width = self.src_width; + (*par).height = self.src_height; + (*par).time_base = ff::AVRational { num: 1, den: 1_000_000 }; + (*par).hw_frames_ctx = ff::av_buffer_ref(self.va_map_frames); + let set = ff::av_buffersrc_parameters_set(self.buffersrc_ctx, par); + ff::av_free(par as *mut _); + if set < 0 { + return Err(format!("av_buffersrc_parameters_set: {}", ff::err_to_string(set))); + } + let inited = ff::avfilter_init_str(self.buffersrc_ctx, ptr::null()); + if inited < 0 { + return Err(format!("avfilter_init_str(buffersrc): {}", ff::err_to_string(inited))); + } + + let rc = ff::avfilter_graph_create_filter( + &mut self.buffersink_ctx, + buffersink, + c"out".as_ptr(), + ptr::null(), + ptr::null_mut(), + self.graph, + ); + if rc < 0 { + return Err(format!("create buffersink: {}", ff::err_to_string(rc))); + } + + // Output size = the recorded (out) size. For a monitor that equals the + // source; for a window it is the crop rectangle, and the per-frame crop + // fields set in `import` pick which region of the source is scaled into it. + let scale_args = std::ffi::CString::new(format!( + "w={}:h={}:format=nv12", + self.out_width, self.out_height + )) + .map_err(|_| "scale_vaapi args contained a NUL".to_owned())?; + let mut scale_ctx: *mut ff::AVFilterContext = ptr::null_mut(); + let rc = ff::avfilter_graph_create_filter( + &mut scale_ctx, + scale, + c"vpp".as_ptr(), + scale_args.as_ptr(), + ptr::null_mut(), + self.graph, + ); + if rc < 0 { + return Err(format!("create scale_vaapi: {}", ff::err_to_string(rc))); + } + // scale_vaapi needs a device to allocate its NV12 output pool; take it + // from the shared VAAPI device rather than relying on propagation. + (*scale_ctx).hw_device_ctx = ff::av_buffer_ref(self.va_device); + + let rc = ff::avfilter_link(self.buffersrc_ctx, 0, scale_ctx, 0); + if rc < 0 { + return Err(format!("avfilter_link(in->vpp): {}", ff::err_to_string(rc))); + } + let rc = ff::avfilter_link(scale_ctx, 0, self.buffersink_ctx, 0); + if rc < 0 { + return Err(format!("avfilter_link(vpp->out): {}", ff::err_to_string(rc))); + } + + let rc = ff::avfilter_graph_config(self.graph, ptr::null_mut()); + if rc < 0 { + return Err(format!("avfilter_graph_config: {}", ff::err_to_string(rc))); + } + Ok(()) + } + + /// The NV12 VAAPI frames context the graph emits into — the encoder opens + /// against this so it accepts the surfaces [`Self::import`] returns. + pub fn output_frames_ctx(&self) -> *mut ff::AVBufferRef { + // SAFETY: valid after a successful `build_graph`; the sink has one input. + unsafe { ff::av_buffersink_get_hw_frames_ctx(self.buffersink_ctx) } + } + + /// The shared VAAPI device, for the encoder's `hwaccel` context. + pub fn device(&self) -> *mut ff::AVBufferRef { + self.va_device + } + + /// Maps one dmabuf and returns an NV12 VAAPI frame (caller unrefs it). The + /// plane fds are only touched during this call. `crop_x`/`crop_y` are the + /// origin of the recorded region within the source; the region size is the + /// importer's output size. For a monitor both are 0 and out == src (no crop). + pub fn import( + &mut self, + frame: &DmabufFrame, + crop_x: i32, + crop_y: i32, + ) -> Result<*mut ff::AVFrame, String> { + if frame.planes.is_empty() || frame.planes.len() > 4 { + return Err(format!("dmabuf has {} planes", frame.planes.len())); + } + // We build a single DRM object from planes[0]'s fd and point every plane at + // it, so all planes must be backed by that one fd. Each plane now owns its + // own dup (see `DmabufPlane`), so two planes aliasing one buffer no longer + // share a NUMBER — this therefore accepts only the single-plane case. That is + // exactly our RGB formats; a genuine multi-plane buffer (never negotiated) is + // conservatively rejected rather than risk VAAPI reading the wrong memory. + if frame + .planes + .iter() + .any(|plane| plane.fd.as_raw_fd() != frame.planes[0].fd.as_raw_fd()) + { + return Err("dmabuf planes span multiple fds, which this importer does not handle".to_owned()); + } + // SAFETY: every allocated frame/buffer is freed on the error paths and on + // success ownership of the NV12 frame passes to the caller. + unsafe { + // The DRM descriptor must outlive the mapped frame: av_hwframe_map + // retains `src` (ref-counted) until the mapping is released, so a + // stack descriptor would dangle once this function returns. Allocate + // it on the heap and free it from the AVBufferRef's own callback. + let desc = ff::av_mallocz(std::mem::size_of::()) + as *mut ff::AVDRMFrameDescriptor; + if desc.is_null() { + return Err("av_mallocz(drm descriptor) failed".to_owned()); + } + (*desc).nb_objects = 1; + (*desc).objects[0].fd = frame.planes[0].fd.as_raw_fd(); + (*desc).objects[0].size = 0; // recovered by the driver from the fd + (*desc).objects[0].format_modifier = frame.modifier; + (*desc).nb_layers = 1; + (*desc).layers[0].format = frame.drm_fourcc; + (*desc).layers[0].nb_planes = frame.planes.len() as i32; + for (i, plane) in frame.planes.iter().enumerate() { + (*desc).layers[0].planes[i].object_index = 0; + (*desc).layers[0].planes[i].offset = plane.offset as isize; + (*desc).layers[0].planes[i].pitch = plane.stride as isize; + } + + let src = ff::av_frame_alloc(); + if src.is_null() { + ff::av_free(desc as *mut std::ffi::c_void); + return Err("av_frame_alloc(src) failed".to_owned()); + } + (*src).format = ff::AV_PIX_FMT_DRM_PRIME as i32; + (*src).width = self.src_width; + (*src).height = self.src_height; + // av_hwframe_map needs a ref-counted source; wrap the heap descriptor + // in an AVBufferRef that frees it when the last reference drops (which + // is after the mapped frame that retains `src` is released). + let buf = ff::av_buffer_create( + desc as *mut u8, + std::mem::size_of::(), + Some(drm_descriptor_free), + ptr::null_mut(), + 0, + ); + if buf.is_null() { + ff::av_free(desc as *mut std::ffi::c_void); + free_frame(src); + return Err("av_buffer_create(drm descriptor) failed".to_owned()); + } + (*src).buf[0] = buf; + (*src).data[0] = (*buf).data; + (*src).hw_frames_ctx = ff::av_buffer_ref(self.drm_frames); + + // Map the dmabuf into a VAAPI (BGRx) surface, zero-copy. + let mapped = ff::av_frame_alloc(); + if mapped.is_null() { + free_frame(src); + return Err("av_frame_alloc(mapped) failed".to_owned()); + } + (*mapped).format = ff::AV_PIX_FMT_VAAPI as i32; + (*mapped).hw_frames_ctx = ff::av_buffer_ref(self.va_map_frames); + let mrc = ff::av_hwframe_map( + mapped, + src, + (ff::AV_HWFRAME_MAP_DIRECT | ff::AV_HWFRAME_MAP_READ) as i32, + ); + // `src` (and thus `desc`) is no longer needed once mapped. + free_frame(src); + if mrc < 0 { + free_frame(mapped); + return Err(format!("av_hwframe_map: {}", ff::err_to_string(mrc))); + } + + // Crop the source down to the recorded region at the live origin. + // scale_vaapi reads these fields to set the VA source rectangle, so a + // window is cropped on the GPU before scaling. A monitor leaves them + // at 0 (crop_x/y are 0 and out == src), so nothing is cropped. + (*mapped).crop_left = crop_x.max(0) as usize; + (*mapped).crop_top = crop_y.max(0) as usize; + (*mapped).crop_right = (self.src_width - crop_x - self.out_width).max(0) as usize; + (*mapped).crop_bottom = (self.src_height - crop_y - self.out_height).max(0) as usize; + + // Push through scale_vaapi → NV12. + let pushed = ff::av_buffersrc_add_frame(self.buffersrc_ctx, mapped); + free_frame(mapped); + if pushed < 0 { + return Err(format!("av_buffersrc_add_frame: {}", ff::err_to_string(pushed))); + } + + let nv12 = ff::av_frame_alloc(); + if nv12.is_null() { + return Err("av_frame_alloc(nv12) failed".to_owned()); + } + let got = ff::av_buffersink_get_frame(self.buffersink_ctx, nv12); + if got < 0 { + free_frame(nv12); + return Err(format!("av_buffersink_get_frame: {}", ff::err_to_string(got))); + } + Ok(nv12) + } + } +} + +impl Drop for DmabufImporter { + fn drop(&mut self) { + // SAFETY: each pointer is freed once; nulls are ignored by the ffmpeg + // frees, and the order is graph → frames → devices. + unsafe { + if !self.graph.is_null() { + ff::avfilter_graph_free(&mut self.graph); + } + for frames in [&mut self.drm_frames, &mut self.va_map_frames] { + if !frames.is_null() { + ff::av_buffer_unref(frames); + } + } + for device in [&mut self.drm_device, &mut self.va_device] { + if !device.is_null() { + ff::av_buffer_unref(device); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Pins the DRM-fourcc → ffmpeg-format mapping. The fourcc constants are + // hand-duplicated between this file and the C shim's `osc_spa_format_to_drm_fourcc`, + // so a transposed XBGR/XRGB would silently swap red and blue in every recording + // with nothing else to catch it. The fourccs are spelled out here independently + // of `sw_format_for_fourcc`'s own consts so the two must agree. + #[test] + fn maps_each_negotiated_fourcc_to_its_packed_format() { + // "XR24" (BGRx) → BGR0, "AR24" (BGRA) → BGRA, + // "XB24" (RGBx) → 0BGR, "AB24" (RGBA) → ABGR. + assert_eq!(sw_format_for_fourcc(0x3432_5258), Some(ff::AV_PIX_FMT_BGR0)); + assert_eq!(sw_format_for_fourcc(0x3432_5241), Some(ff::AV_PIX_FMT_BGRA)); + assert_eq!(sw_format_for_fourcc(0x3432_4258), Some(ff::AV_PIX_FMT_0BGR)); + assert_eq!(sw_format_for_fourcc(0x3432_4241), Some(ff::AV_PIX_FMT_ABGR)); + } + + #[test] + fn rejects_a_fourcc_we_do_not_negotiate() { + // "NV12" is a real fourcc, just not one of our packed RGB layouts. + assert_eq!(sw_format_for_fourcc(0x3231_564e), None); + assert_eq!(sw_format_for_fourcc(0), None); + } +} diff --git a/electron/native/pipewire-capture/src/encoder.rs b/electron/native/pipewire-capture/src/encoder.rs index e574ae405..8e197dddd 100644 --- a/electron/native/pipewire-capture/src/encoder.rs +++ b/electron/native/pipewire-capture/src/encoder.rs @@ -33,6 +33,18 @@ pub enum Backend { Software, } +/// Whether the dmabuf → VAAPI zero-copy import may be used for a `forced` encoder +/// choice. The import can only feed VAAPI, so `software`/`vulkan` must skip it — +/// otherwise the documented `OPENSCREEN_LINUX_ENCODER` override is silently +/// ignored on the dmabuf path. +/// +/// The SINGLE source of truth for this: the negotiation offer (`prefer_dmabuf` in +/// main) and the importer build (`Capture::start`) both call it, so they cannot +/// drift into offering dmabuf that no importer will consume. +pub fn forced_allows_dmabuf(forced: Option) -> bool { + matches!(forced, None | Some(Backend::Vaapi)) +} + impl Backend { /// The name the app reports and the tests match on. Kept in the same /// vocabulary as the Windows helper's `encoder-selection` event. @@ -193,6 +205,11 @@ pub struct VideoEncoder { sw_frame: *mut ff::AVFrame, /// The GPU-side frame handed to a hardware encoder. Null for software. hw_frame: *mut ff::AVFrame, + /// A ready-to-encode VAAPI NV12 surface produced by the dmabuf importer + /// (issue #507). When non-null it is sent directly — no sws_scale, no upload — + /// and held across the clock-driven re-encodes until the next frame replaces + /// it. Null on the shm/software path. + hw_staged: *mut ff::AVFrame, sws: *mut ff::SwsContext, sws_src_format: ff::AVPixelFormat, packet: *mut ff::AVPacket, @@ -235,7 +252,7 @@ impl VideoEncoder { failures.push(format!("{}: {reason}", backend.as_str())); continue; } - match Self::open_backend(backend, ¶ms) { + match Self::open_backend(backend, ¶ms, None) { Ok(encoder) => return Ok(encoder), Err(error) => { on_attempt(backend, &error); @@ -252,7 +269,26 @@ impl VideoEncoder { )) } - fn open_backend(backend: Backend, params: &VideoParams) -> Result { + /// Opens the VAAPI encoder to consume surfaces from an EXISTING device and + /// NV12 frames pool — the ones the dmabuf importer built. Sharing the pool is + /// what lets `encode_staged` send an imported surface straight to + /// `avcodec_send_frame` without a copy (issue #507). + /// + /// SAFETY: `device` and `frames_ctx` must be a live VAAPI device and an NV12 + /// VAAPI frames context on it; the encoder takes its own references. + pub unsafe fn open_importing( + params: VideoParams, + device: *mut ff::AVBufferRef, + frames_ctx: *mut ff::AVBufferRef, + ) -> Result { + Self::open_backend(Backend::Vaapi, ¶ms, Some((device, frames_ctx))) + } + + fn open_backend( + backend: Backend, + params: &VideoParams, + external: Option<(*mut ff::AVBufferRef, *mut ff::AVBufferRef)>, + ) -> Result { // SAFETY: this whole function is a single ffmpeg setup sequence. Every // allocation is stored in `encoder` as soon as it succeeds, so the Drop // impl frees whatever was reached if a later step fails. @@ -277,6 +313,7 @@ impl VideoEncoder { hw_frames: ptr::null_mut(), sw_frame: ptr::null_mut(), hw_frame: ptr::null_mut(), + hw_staged: ptr::null_mut(), sws: ptr::null_mut(), sws_src_format: ff::AV_PIX_FMT_NONE, packet: ptr::null_mut(), @@ -320,7 +357,7 @@ impl VideoEncoder { (*codec_ctx).flags |= ff::AV_CODEC_FLAG_GLOBAL_HEADER as i32; if let Some(device_type) = backend.hw_device_type() { - encoder.attach_hardware(device_type, params)?; + encoder.attach_hardware(device_type, params, external)?; } let opened = ff::avcodec_open2(codec_ctx, codec, ptr::null_mut()); @@ -350,7 +387,36 @@ impl VideoEncoder { &mut self, device_type: ff::AVHWDeviceType, params: &VideoParams, + external: Option<(*mut ff::AVBufferRef, *mut ff::AVBufferRef)>, ) -> Result<(), String> { + // The dmabuf importer already built a VAAPI device and an NV12 pool; the + // encoder must consume from THAT pool, so take references to it instead of + // creating a second, incompatible one. See `open_importing`. + if let Some((device, frames_ctx)) = external { + // The guard belongs on the INPUTS, before the refs are taken: + // av_buffer_ref dereferences its argument (`*ret = *buf`), so a null + // one faults inside ffmpeg instead of coming back as a null return + // value the old check could inspect. And null IS reachable here -- + // `DmabufImporter::output_frames_ctx` is av_buffersink_get_hw_frames_ctx, + // which returns NULL whenever the sink's input link carries no hw + // frames context. + if device.is_null() || frames_ctx.is_null() { + return Err("av_buffer_ref on the shared VAAPI context returned null".to_owned()); + } + self.hw_device = ff::av_buffer_ref(device); + self.hw_frames = ff::av_buffer_ref(frames_ctx); + // The results still get checked: av_buffer_ref allocates, and the + // hw_frames_ctx ref below would dereference whatever it handed back. + if self.hw_device.is_null() || self.hw_frames.is_null() { + return Err("av_buffer_ref on the shared VAAPI context returned null".to_owned()); + } + (*self.codec_ctx).hw_frames_ctx = ff::av_buffer_ref(self.hw_frames); + if (*self.codec_ctx).hw_frames_ctx.is_null() { + return Err("av_buffer_ref on the shared frames context returned null".to_owned()); + } + return Ok(()); + } + let created = ff::av_hwdevice_ctx_create( &mut self.hw_device, device_type, @@ -493,14 +559,38 @@ impl VideoEncoder { if scaled < 0 { return Err(format!("sws_scale: {}", ff::err_to_string(scaled))); } + // This CPU frame is now what `encode_staged` must send, so drop any hw + // surface a previous dmabuf frame left staged — e.g. after the stream + // renegotiates to a modifier-less format and frames start arriving on + // the sws path. `encode_staged` prefers `hw_staged` whenever it is + // non-null, so without this it would keep re-sending that stale surface + // and freeze the video. `av_frame_free` nulls the pointer. + if !self.hw_staged.is_null() { + ff::av_frame_free(&mut self.hw_staged); + } self.staged = true; } Ok(()) } - /// True once [`Self::stage`] has put a picture in the staging buffer. Before - /// that there is nothing to encode and [`Self::encode_staged`] would emit a - /// frame of uninitialised memory. + /// Stages a ready VAAPI NV12 surface produced by the dmabuf importer. Takes + /// ownership of `frame`; the previous one is released. Unlike [`Self::stage`] + /// there is no conversion or upload — the surface is encoded as-is and held + /// across the clock-driven re-encodes until the next frame replaces it. + /// + /// SAFETY: `frame` must be a valid VAAPI NV12 `AVFrame` from the shared pool + /// the encoder was opened against (see `open_importing`). + pub unsafe fn stage_hw(&mut self, frame: *mut ff::AVFrame) { + if !self.hw_staged.is_null() { + let mut old = self.hw_staged; + ff::av_frame_free(&mut old); + } + self.hw_staged = frame; + self.staged = true; + } + + /// True once a picture has been staged — via [`Self::stage`] (shm/software) + /// or [`Self::stage_hw`] (dmabuf). Before that there is nothing to encode. pub fn has_staged_frame(&self) -> bool { self.staged } @@ -521,7 +611,13 @@ impl VideoEncoder { // `sw_frame`, and every pointer below is owned by `self`. unsafe { let upload_started = std::time::Instant::now(); - let frame = if self.hw_frames.is_null() { + let mut used_upload = false; + let frame = if !self.hw_staged.is_null() { + // Imported dmabuf surface: already NV12 on the GPU. No upload, no + // conversion — just timestamp it. Held for the next re-encode. + (*self.hw_staged).pts = pts; + self.hw_staged + } else if self.hw_frames.is_null() { (*self.sw_frame).pts = pts; self.sw_frame } else { @@ -540,6 +636,7 @@ impl VideoEncoder { )); } (*self.hw_frame).pts = pts; + used_upload = true; self.hw_frame }; self.stats.upload_ns += upload_started.elapsed().as_nanos(); @@ -549,11 +646,13 @@ impl VideoEncoder { self.stats.encode_ns += encode_started.elapsed().as_nanos(); self.stats.frames += 1; - if !self.hw_frame.is_null() { - // Release our reference to the GPU surface; the encoder keeps - // its own for as long as it needs one. Without this the pool - // drains after `initial_pool_size` frames and every subsequent - // av_hwframe_get_buffer blocks. + if used_upload { + // Release our reference to the per-encode upload surface; the + // encoder keeps its own for as long as it needs one. Without this + // the pool drains after `initial_pool_size` frames and every + // subsequent av_hwframe_get_buffer blocks. The imported + // `hw_staged` surface is NOT released here — it is held for the + // next clock-driven re-encode and freed in `stage_hw`/`Drop`. ff::av_frame_unref(self.hw_frame); } } @@ -697,6 +796,9 @@ impl Drop for VideoEncoder { if !self.hw_frame.is_null() { ff::av_frame_free(&mut self.hw_frame); } + if !self.hw_staged.is_null() { + ff::av_frame_free(&mut self.hw_staged); + } if !self.sw_frame.is_null() { ff::av_frame_free(&mut self.sw_frame); } diff --git a/electron/native/pipewire-capture/src/events.rs b/electron/native/pipewire-capture/src/events.rs index 085ea56aa..ce98ac87e 100644 --- a/electron/native/pipewire-capture/src/events.rs +++ b/electron/native/pipewire-capture/src/events.rs @@ -97,6 +97,12 @@ pub enum Event { asset_id: Option, #[serde(skip_serializing_if = "Option::is_none")] asset: Option, + /// `"click"` on the sample that coincides with a left-button press read + /// from evdev (see `input.rs`), absent otherwise. Omitted rather than + /// defaulted to `"move"` so the accumulator keeps that fallback in one + /// place and the wire stays quiet on the common case. + #[serde(skip_serializing_if = "Option::is_none")] + interaction_type: Option, }, /// Which capture node each audio source was linked to. /// @@ -303,12 +309,32 @@ mod tests { visible: true, asset_id: None, asset: None, + interaction_type: None, }); assert_eq!(value["event"], "cursor-sample"); assert_eq!(value["x"], 100); assert_eq!(value["visible"], true); assert!(value.get("assetId").is_none()); assert!(value.get("asset").is_none()); + // A plain move stays silent about its interaction so the accumulator's + // "move" default is the single source of that word. + assert!(value.get("interactionType").is_none()); + } + + #[test] + fn cursor_samples_report_a_click_when_tagged() { + let value = parse_one(&Event::CursorSample { + timestamp_ms: 12, + x: 100, + y: 200, + width: 1920, + height: 1080, + visible: true, + asset_id: None, + asset: None, + interaction_type: Some("click".to_owned()), + }); + assert_eq!(value["interactionType"], "click"); } #[test] @@ -329,6 +355,7 @@ mod tests { hotspot_x: 4, hotspot_y: 3, }), + interaction_type: None, }); assert_eq!(value["assetId"], "abc"); assert_eq!(value["asset"]["imageDataUrl"], "data:image/png;base64,AA=="); @@ -359,6 +386,7 @@ mod tests { visible: true, asset_id: None, asset: None, + interaction_type: None, }); assert_eq!(value["timestampMs"], 1234, "a sample's capture time must not be overwritten"); } diff --git a/electron/native/pipewire-capture/src/input.rs b/electron/native/pipewire-capture/src/input.rs new file mode 100644 index 000000000..a1d41c90d --- /dev/null +++ b/electron/native/pipewire-capture/src/input.rs @@ -0,0 +1,211 @@ +//! Left mouse-button telemetry on Wayland, read from evdev. +//! +//! WHY THIS EXISTS. Wayland deliberately denies an unprivileged process any view +//! of global input: the ScreenCast portal reports cursor POSITION as frame +//! metadata but never button state, and the only portal that streams input +//! (`InputCapture`) *grabs* it, redirecting clicks away from the app being +//! recorded — useless while the user is demoing. The one remaining source is the +//! kernel's evdev interface (`/dev/input/event*`). Reading it needs membership in +//! the `input` group (the nodes are `root:input`), which is the user's own, +//! out-of-band act of consent — the Wayland equivalent of the button state the +//! macOS and Windows helpers already read from their native APIs. +//! +//! SCOPE AND PRIVACY. A pointer node can also deliver keystrokes on a combined +//! keyboard+mouse device. This reader inspects ONLY `EV_KEY` events whose code is +//! `BTN_LEFT`, and only their press edge; it never reads, stores, or forwards any +//! other key code. Enumerating the devices does briefly open each readable +//! `/dev/input/event*` node to inspect its capability bits, but any node that +//! does not advertise `BTN_LEFT` is dropped immediately, without a single event +//! ever being read from it — only `BTN_LEFT` devices get a reader. Set +//! `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` to turn it off entirely even where the +//! permission exists. + +use std::collections::HashSet; +use std::io::ErrorKind; +use std::path::PathBuf; +use std::sync::mpsc::Sender; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +use evdev::{Device, EventType, KeyCode}; + +use crate::events::timestamp_ms; +use crate::Message; + +const DISABLE_ENV: &str = "OPENSCREEN_DISABLE_CLICK_CAPTURE"; + +/// How often the hotplug watcher re-scans `/dev/input` for pointer devices that +/// appeared after startup. A few seconds is imperceptible for a device the user +/// just plugged in and costs a cheap directory walk. +const RESCAN_INTERVAL: Duration = Duration::from_secs(3); + +/// True when this evdev event is the press edge of the left mouse button. +/// +/// Extracted as a pure function so the decision is unit-testable without a real +/// device: a release (`value == 0`), an autorepeat (`value == 2`), and every +/// non-`BTN_LEFT` code — including every keyboard key — must NOT count. +pub fn is_left_button_press(event_type: EventType, code: u16, value: i32) -> bool { + event_type == EventType::KEY && code == KeyCode::BTN_LEFT.0 && value == 1 +} + +/// What [`spawn_readers`] settled on — the caller uses it to tell the log why +/// Linux clicks are or are not being captured. +/// +/// The two ways of ending up without clicks are NOT the same event: no readable +/// node is a permission the operator may still want to grant, while `DISABLE_ENV` +/// is one they explicitly declined — recommending the `input` group there answers +/// a question nobody asked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClickCapture { + /// Turned off by `OPENSCREEN_DISABLE_CLICK_CAPTURE`; nothing was opened. + Disabled, + /// No readable `/dev/input` node reports `BTN_LEFT` — the common case, when + /// the user is not in the `input` group. + NoDevice, + /// At least one device is open, with a reader thread running on it. + Active, +} + +/// Opens every readable pointer device that reports `BTN_LEFT`, spawns a reader +/// thread per device, and leaves a daemon thread re-scanning for devices plugged +/// in later. +/// +/// Never fails: an unreadable node (the common case, when the user is not in the +/// `input` group) is skipped by `evdev::enumerate`, and no readable node at all +/// simply means every sample stays `"move"`, exactly as before this existed. The +/// returned value reflects the INITIAL scan only — a device attached afterwards +/// is adopted by the watcher without changing it. +pub fn spawn_readers(sender: &Sender) -> ClickCapture { + if std::env::var_os(DISABLE_ENV).is_some() { + return ClickCapture::Disabled; + } + // Paths with a LIVE reader. Shared with the reader threads: each removes its + // own path when it exits, so a device that unplugs and reconnects on the SAME + // node path is adopted again by the next scan. A set that only grew skipped + // such a replug for the rest of the recording. + let opened: Arc>> = Arc::new(Mutex::new(HashSet::new())); + scan_once(sender, &opened); + let result = if opened.lock().unwrap().is_empty() { + ClickCapture::NoDevice + } else { + ClickCapture::Active + }; + // Hotplug: the one-shot scan above cannot see a mouse attached mid-recording, + // so a daemon thread re-scans and starts readers for nodes it has not seen. + // Detached, like the reader threads — it ends when the process does. + let watch_sender = sender.clone(); + let watch_opened = Arc::clone(&opened); + thread::spawn(move || loop { + thread::sleep(RESCAN_INTERVAL); + scan_once(&watch_sender, &watch_opened); + }); + result +} + +/// Spawns a reader for every `BTN_LEFT` device not already being read, recording +/// each newly opened node's path. Shared by the initial scan and the watcher; the +/// check-and-insert is one locked step so two scans cannot both adopt one path. +fn scan_once(sender: &Sender, opened: &Arc>>) { + for (path, device) in evdev::enumerate() { + if !device_reports_left_button(&device) { + continue; + } + if !opened.lock().unwrap().insert(path.clone()) { + continue; // already has a live reader + } + let forward = sender.clone(); + let owned = Arc::clone(opened); + thread::spawn(move || read_device(device, path, forward, owned)); + } +} + +/// A touchpad advertises `BTN_LEFT` for a physical clickpad press, so it passes +/// this check and its node is opened — but tap-to-click never arrives here. +/// libinput consumes the raw `BTN_TOUCH`/`ABS_MT_*` stream and synthesises the +/// button for its own clients without writing `BTN_LEFT` back to the kernel +/// device, so a tap is invisible at the evdev layer we read. The consequence is +/// documented for users under "Mouse clicks on Wayland" in the installation docs: +/// on a touchpad only hard presses are captured, taps are not. +fn device_reports_left_button(device: &Device) -> bool { + device + .supported_keys() + .is_some_and(|keys| keys.contains(KeyCode::BTN_LEFT)) +} + +/// Blocks reading `device`, forwarding one `PointerButton` message per left-button +/// press, each stamped with the press time so a click is not backdated to the +/// next cursor sample. Returns when the device fails terminally (e.g. unplugged) +/// or the loop's channel has closed, so the thread cannot outlive the recording +/// it serves. A transient `EINTR` is retried, not mistaken for an unplug. +/// +/// On EVERY exit it drops `path` from `opened`, so a device reconnecting on the +/// same node path is re-adopted by the next scan. +fn read_device( + mut device: Device, + path: PathBuf, + sender: Sender, + opened: Arc>>, +) { + let release = || { + opened.lock().unwrap().remove(&path); + }; + loop { + let events = match device.fetch_events() { + Ok(events) => events, + // A signal interrupted the blocking read — not a device failure. + Err(err) if err.kind() == ErrorKind::Interrupted => continue, + // A real error, typically the device unplugging. Report it rather + // than ending silently, so click capture going quiet mid-recording + // is answerable from the log; the watcher re-adopts it on a replug. + Err(err) => { + release(); + let _ = sender.send(Message::PointerDeviceLost(err.to_string())); + return; + } + }; + for event in events { + if is_left_button_press(event.event_type(), event.code(), event.value()) + && sender.send(Message::PointerButton(timestamp_ms())).is_err() + { + release(); + return; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const BTN_LEFT: u16 = KeyCode::BTN_LEFT.0; + const BTN_RIGHT: u16 = KeyCode::BTN_RIGHT.0; + + #[test] + fn a_left_button_press_is_a_click() { + assert!(is_left_button_press(EventType::KEY, BTN_LEFT, 1)); + } + + #[test] + fn a_left_button_release_is_not() { + assert!(!is_left_button_press(EventType::KEY, BTN_LEFT, 0)); + } + + #[test] + fn a_left_button_autorepeat_is_not() { + // A held button emits value 2; only the 0->1 edge is a click. + assert!(!is_left_button_press(EventType::KEY, BTN_LEFT, 2)); + } + + #[test] + fn a_right_button_press_is_not_a_left_click() { + assert!(!is_left_button_press(EventType::KEY, BTN_RIGHT, 1)); + } + + #[test] + fn a_key_matching_btn_lefts_code_on_another_axis_is_not_a_click() { + // Same numeric code but a relative-motion event, not a key — must miss. + assert!(!is_left_button_press(EventType::RELATIVE, BTN_LEFT, 1)); + } +} diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 8ec4078c2..5da1877e3 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -20,15 +20,20 @@ //! the cursor-only session Stage 1 shipped, which is what //! `PipeWireCursorRecordingSession` still uses. //! -//! WHAT IT CANNOT DO. Mouse buttons. Wayland exposes no portal for input -//! events, and /dev/input/event* is root:input. Every sample is therefore a -//! "move"; there is no click detection to be had here at any effort level. +//! MOUSE BUTTONS. Not from the portal — Wayland exposes no portal for input +//! events. The one source left is evdev (/dev/input/event*), which is root:input +//! and so needs the user in the `input` group. When that permission exists the +//! helper reads left-button presses and tags the coinciding sample "click"; when +//! it does not, every sample is a "move", as before. See `input.rs` for the +//! reader and its deliberately narrow scope (BTN_LEFT only, never keystrokes). mod bitmap; mod capture; +mod dmabuf_import; mod encoder; mod events; mod ffmpeg; +mod input; mod portal; mod shim; @@ -98,6 +103,12 @@ const MAX_FRAMES_AWAITING_CROP: u32 = 8; /// How much audio may queue before the oldest is discarded. Generous: the drain /// runs every loop tick, so reaching this means the encoder stopped entirely. const AUDIO_RING_SECONDS: usize = 2; +/// How many dmabuf imports may fail in a row before the recording gives up. One +/// failure is a recoverable dropped frame (the previous is held forward), but this +/// many in a row means the import path is broken — typically a render node that +/// cannot map the compositor's GPU buffers — and the file would otherwise be empty +/// behind a wall of `frame-dropped` warnings. ~1–2 s at 30–60 fps. +const MAX_CONSECUTIVE_IMPORT_FAILURES: u32 = 60; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "camelCase", default)] @@ -208,6 +219,16 @@ struct AudioSourceConfig { enum Message { Portal(Box>), Stream(StreamEvent), + /// A left mouse-button press observed on evdev, carrying the press time in + /// milliseconds so the emitted click sample is stamped when it happened + /// rather than at the next throttled sample. See [`input`] for why this is + /// the only way to see a button on Wayland, and for its permission and + /// privacy model. + PointerButton(u64), + /// A pointer reader thread stopped on a device error (typically an unplug); + /// the string is the OS error. Surfaced as a warning so click capture going + /// quiet mid-recording is not silent. + PointerDeviceLost(String), /// Arm a deferred session: connect to PipeWire and start encoding. Record, Pause, @@ -283,6 +304,23 @@ fn main() { let (sender, receiver) = mpsc::channel::(); spawn_stdin_reader(sender.clone()); spawn_portal(sender.clone(), cursor_mode); + // Left-button telemetry from evdev. Gated on the cursor mode FIRST: a session + // that paints no cursor emits no samples to tag, so opening every pointer node + // and blocking a thread on each would buy nothing. Warn only when the devices + // are unreadable — never when the operator opted out through the env var, + // which would recommend the permission they just declined — so a "clicks do + // nothing on Linux" report is answerable from the log alone rather than + // looking like a capture bug. + if cursor_mode.reports_cursor() + && input::spawn_readers(&sender) == input::ClickCapture::NoDevice + { + let _ = emitter.emit(&Event::Warning { + code: "click-capture-unavailable".to_owned(), + message: "no readable /dev/input pointer device — add this user to the 'input' \ + group to record click telemetry; cursor samples will otherwise all be moves" + .to_owned(), + }); + } let session = RunConfig { tick, @@ -505,6 +543,7 @@ fn begin_stream( portal_stream: &mut Option, granted_kind: &mut Option, stream: portal::PortalStream, + prefer_dmabuf: bool, ) -> Result<(), ()> { // The fd is consumed by libpipewire; the rest is kept for the // `stream-started` event, emitted once the format is negotiated. @@ -517,6 +556,7 @@ fn begin_stream( let _ = forward.send(Message::Stream(event)); }), frames.clone(), + prefer_dmabuf, ) { Ok(started) => { *session = Some(started); @@ -568,6 +608,17 @@ fn run( let mut cursor: Option = None; let mut known_assets: HashSet = HashSet::new(); let mut pending_asset: Option = None; + // Wall-clock ms at which the pw_stream last reached `streaming` (mutter began + // handing us frames), or `None` when it is not streaming. A press counts only + // if its own read time is at or after this: before streaming the only thing on + // screen is the portal picker, and the click that dismisses it (its "Share" + // button) would otherwise ride out on the first sample as a phantom click at + // t≈0. Comparing timestamps rather than a bare flag closes two gaps: the press + // and the stream-state arrive on DIFFERENT channels, so a pre-stream press can + // be dequeued after the flag flips (it is still dropped, its time is older); + // and clearing it on disconnect stops clicks emitting against a stale cursor + // after capture has stopped. + let mut streaming_since: Option = None; let mut reported_cursor_meta = false; // Allocated up front so the PipeWire callback has somewhere to put frames // from the very first buffer; `None` in cursor-only mode, which is also what @@ -576,6 +627,21 @@ fn run( .output_path .as_ref() .map(|_| Arc::new(FrameMailbox::default())); + // Offer dmabuf ahead of shm (issue #507) only for a video session, only when + // the VAAPI import pipeline actually builds on this GPU, AND only when the + // forced-encoder choice can consume it — `software`/`vulkan` cannot, and + // offering dmabuf they can't import makes the first frame fail. The available() + // probe constructs a VAAPI device and filtergraph once here, so a machine that + // cannot import keeps the shm path with no per-recording cost. + let prefer_dmabuf = frames.is_some() + && crate::dmabuf_import::available() + && encoder::forced_allows_dmabuf(config.forced_encoder); + if frames.is_some() { + let _ = emitter.emit(&Event::Debug { + code: "dmabuf-import".to_owned(), + data: json_map([("available", prefer_dmabuf.into())]), + }); + } let mut capture: Option = None; // Started before the portal picker so the streams are warm and the graph // has settled by the time the first video frame arrives. Everything they @@ -592,11 +658,54 @@ fn run( .checked_sub(config.sample_interval) .unwrap_or_else(Instant::now); let mut exit_code = 0; + // Consecutive dmabuf import failures; reset on any staged frame. A run of them + // means the import never works, which would otherwise record nothing — see + // MAX_CONSECUTIVE_IMPORT_FAILURES. + let mut consecutive_drops: u32 = 0; loop { + // Return PipeWire buffers whose dmabuf imports completed last iteration + // (or that were superseded on the capture thread). This MUST run on this + // loop, not the PipeWire thread — `Session::requeue` takes the thread-loop + // lock, which would deadlock from inside the loop. The one-tick delay is + // harmless: the import has already copied the pixels, so the buffer is + // free, and the pool has other buffers in flight meanwhile. + if let (Some(session), Some(mailbox)) = (session.as_ref(), frames.as_ref()) { + for handle in mailbox.drain_requeue() { + session.requeue(handle); + } + } + match receiver.recv_timeout(config.tick) { Ok(Message::Stop) => break, + // Emit a click sample straight away at the current cursor position, + // stamped with the press time the reader captured: immediate so it is + // not backdated to the next throttled sample, and one sample per press + // so a rapid double-click reads as two clicks rather than collapsing + // into one. Counted only if the press happened at or after streaming + // began (see `streaming_since`): a press while the picker is still up — + // the click on its "Share" button — is older, so it is dropped even if + // its message is delivered after the stream-state one. + Ok(Message::PointerButton(press_ms)) => { + if streaming_since.is_some_and(|since| press_ms >= since) { + emit_sample(emitter, &cursor, size, &mut pending_asset, Some(press_ms)); + } + } + + // A reader lost its device (typically an unplug). Report it — if it + // was the only pointer, clicks stop until one is (re)connected, which + // the input watcher will pick up. + Ok(Message::PointerDeviceLost(reason)) => { + let _ = emitter.emit(&Event::Warning { + code: "click-capture-device-lost".to_owned(), + message: format!( + "a pointer device stopped delivering clicks ({reason}); if it was the \ + only one, clicks are not captured until a device is reconnected" + ), + }); + } + Ok(Message::Pause) => { paused = true; if let Some(capture) = capture.as_mut() { @@ -692,6 +801,7 @@ fn run( config.bitrate, config.forced_encoder, std::mem::take(&mut audio_sources), + frame.dmabuf.as_ref(), ) { Ok((started, selection)) => { let _ = emitter.emit(&Event::EncoderSelection { @@ -747,15 +857,50 @@ fn run( let staged = capture.stage(&frame); let (width, height) = (frame.crop.width, frame.crop.height); mailbox.recycle(frame.pixels); - if let Err(message) = staged { - let _ = emitter.emit(&Event::Error { - code: "encode-failed".to_owned(), - message, - }); - exit_code = 1; - break; + match staged { + Ok(capture::StageOutcome::Staged) => { + consecutive_drops = 0; + } + // Recoverable: one frame the GPU could not import. Warn so it + // is answerable from the log, then carry on — `advance` holds + // the previous frame — rather than ending the file. But a long + // RUN of failures means the import path is broken and the file + // would be empty, so past the threshold fail loudly with a way + // out (the full auto-downgrade to shm remains a follow-up). + Ok(capture::StageOutcome::Dropped(reason)) => { + consecutive_drops += 1; + if consecutive_drops >= MAX_CONSECUTIVE_IMPORT_FAILURES { + let _ = emitter.emit(&Event::Error { + code: "encode-failed".to_owned(), + message: format!( + "the GPU could not import {consecutive_drops} captured \ + frames in a row ({reason}); the render node likely \ + cannot map the compositor's buffers. Set \ + OPENSCREEN_LINUX_RENDER_NODE to the correct /dev/dri \ + node, or OPENSCREEN_LINUX_ENCODER=software for the CPU \ + path." + ), + }); + exit_code = 1; + break; + } + let _ = emitter.emit(&Event::Warning { + code: "frame-dropped".to_owned(), + message: reason, + }); + } + Err(message) => { + let _ = emitter.emit(&Event::Error { + code: "encode-failed".to_owned(), + message, + }); + exit_code = 1; + break; + } } - if first { + // Only once a frame has actually staged — a first frame that + // dropped leaves capture unstarted, so this waits for a real one. + if first && capture.started() { let _ = emitter.emit(&Event::CaptureStarted { timestamp_ms: timestamp_ms(), path: config @@ -796,6 +941,7 @@ fn run( &mut portal_stream, &mut granted_kind, stream, + prefer_dmabuf, ) { exit_code = 1; break; @@ -854,6 +1000,7 @@ fn run( &mut portal_stream, &mut granted_kind, stream, + prefer_dmabuf, ) { exit_code = 1; break; @@ -965,6 +1112,19 @@ fn run( ("error", error.clone().into()), ]), }); + // Once frames are flowing, presses land on recorded content; the + // picker (and the "Share" click that dismissed it) is behind us. + // Stamped so a press read before this instant is dropped by time, + // and cleared on disconnect so clicks stop emitting against a stale + // cursor after capture ends. Only set on the FIRST streaming edge so + // a transient renegotiation `paused`→`streaming` does not re-arm it. + if state == "streaming" { + if streaming_since.is_none() { + streaming_since = Some(timestamp_ms()); + } + } else if state == "unconnected" { + streaming_since = None; + } if let Some(error) = error { let _ = emitter.emit(&Event::Warning { code: "stream-error".to_owned(), @@ -1052,14 +1212,14 @@ fn run( // A new sprite ships immediately; positions respect the sample // interval so a 120fps compositor cannot flood stdout. if asset_is_new || last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset); + emit_sample(emitter, &cursor, size, &mut pending_asset, None); last_emit = Instant::now(); } } Err(RecvTimeoutError::Timeout) => { if cursor.is_some() && last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset); + emit_sample(emitter, &cursor, size, &mut pending_asset, None); last_emit = Instant::now(); } // The heartbeat that keeps the output at a constant frame rate @@ -1148,18 +1308,30 @@ fn finish_capture( } } +/// Emits one cursor sample at the current position. `click` is `Some(press_ms)` +/// for a left-button press — the sample is then tagged `"click"` and stamped with +/// that press time — or `None` for an ordinary throttled position sample stamped +/// now. Either way a pending new sprite rides out on it. fn emit_sample( emitter: &mut Emitter, cursor: &Option, size: Option<(i32, i32)>, pending_asset: &mut Option, + click: Option, ) { let (Some(state), Some((width, height))) = (cursor, size) else { return; }; let visible = state.x >= 0 && state.y >= 0 && state.x < width && state.y < height; + // A click carries its own press time so the bounce lands when the button went + // down, not up to a sample interval later; the position is the latest known, + // which at the sample cadence has not moved enough to be wrong. + let (timestamp, interaction_type) = match click { + Some(press_ms) => (press_ms, Some("click".to_owned())), + None => (timestamp_ms(), None), + }; let _ = emitter.emit(&Event::CursorSample { - timestamp_ms: timestamp_ms(), + timestamp_ms: timestamp, x: state.x, y: state.y, width, @@ -1167,6 +1339,7 @@ fn emit_sample( visible, asset_id: state.asset_id.clone(), asset: pending_asset.take(), + interaction_type, }); } diff --git a/electron/native/pipewire-capture/src/shim.rs b/electron/native/pipewire-capture/src/shim.rs index 9d3a03ec7..68b58140f 100644 --- a/electron/native/pipewire-capture/src/shim.rs +++ b/electron/native/pipewire-capture/src/shim.rs @@ -10,7 +10,7 @@ //! letting a Rust panic unwind into C. use std::ffi::{c_char, c_void, CStr}; -use std::os::fd::{IntoRawFd, OwnedFd}; +use std::os::fd::{BorrowedFd, IntoRawFd, OwnedFd}; use std::panic::{catch_unwind, AssertUnwindSafe}; #[repr(C)] @@ -46,8 +46,40 @@ pub struct RawFrame { pub crop_width: i32, pub crop_height: i32, pub has_crop: i32, + /// Zero-copy dmabuf hand-off (issue #507). When non-zero, `data` is null and + /// the frame is a tiled GPU buffer described by the fields below — imported + /// as a VAAPI surface rather than read from `data`. Layout mirrors + /// `struct osc_pw_frame` in pw_shim.h exactly. + pub is_dmabuf: i32, + pub modifier: u64, + pub drm_fourcc: u32, + pub n_planes: i32, + pub plane_fd: [i32; 4], + pub plane_offset: [i32; 4], + pub plane_stride: [i32; 4], + /// The `struct pw_buffer *` this frame came from (opaque). Returned to + /// `osc_pw_requeue_buffer` once the dmabuf import has copied the pixels, if + /// `on_frame` took ownership of it. Null/unused on the CPU path. + pub buffer_handle: *mut c_void, + /// Registration generation of `buffer_handle`, handed back with it so the + /// re-queue can reject a stale pointer a renegotiation reused (see the C side). + pub buffer_generation: u64, } +/// A `struct pw_buffer *` we are holding out of PipeWire's queue until its dmabuf +/// content has been imported, tagged with its registration `generation` so the +/// re-queue can tell it from a newer buffer reusing the same slot. Send so it can +/// travel through the mailbox; the pointer is only ever handed back to +/// `osc_pw_requeue_buffer`, never dereferenced on the Rust side. +#[derive(Debug, Clone, Copy)] +pub struct BufferHandle { + pub ptr: *mut c_void, + pub generation: u64, +} +// SAFETY: the pointer is an opaque token owned by libpipewire; Rust neither reads +// nor writes through it, only returns it to the shim's locked requeue. +unsafe impl Send for BufferHandle {} + #[repr(C)] #[derive(Debug, Clone, Copy)] pub struct RawFormat { @@ -63,7 +95,7 @@ struct RawCallbacks { user: *mut c_void, on_format: extern "C" fn(*mut c_void, *const RawFormat), on_cursor: extern "C" fn(*mut c_void, *const RawCursor), - on_frame: extern "C" fn(*mut c_void, *const RawFrame), + on_frame: extern "C" fn(*mut c_void, *const RawFrame) -> i32, on_buffer_info: extern "C" fn(*mut c_void, u32, u32, i32, u32, *const c_char), on_state: extern "C" fn(*mut c_void, *const c_char, *const c_char), } @@ -106,11 +138,17 @@ extern "C" { fd: i32, node_id: u32, want_video: i32, + prefer_dmabuf: i32, callbacks: *const RawCallbacks, err: *mut c_char, err_len: usize, ) -> *mut RawSession; fn osc_pw_stop(session: *mut RawSession); + fn osc_pw_requeue_buffer( + session: *mut RawSession, + buffer_handle: *mut c_void, + buffer_generation: u64, + ); } /// Where stream events go. Called on the PipeWire thread, so it must not block: @@ -180,6 +218,65 @@ pub struct Frame { /// "invalid meta" and "meta covering everything" alike — none of which is a /// reason to crop, and none of which may be guessed apart. pub has_crop: bool, + /// Set for a tiled dmabuf frame (issue #507): `pixels` is empty and the + /// content is on the GPU, described here for a VAAPI import instead. The + /// owned fds close when the frame is dropped or superseded. + pub dmabuf: Option, +} + +/// A tiled dmabuf handed up for GPU import. Holds the PipeWire buffer OUT of the +/// queue (via `buffer_handle`) so the plane fds AND their content stay valid until +/// the import copies the surface — dup'ing the fds alone would preserve the object +/// but not a content snapshot, letting the compositor overwrite a re-queued buffer +/// (CodeRabbit / issue #507). On drop the handle is pushed to `requeue`, which the +/// main loop drains and hands back to the shim's locked re-queue. +pub struct DmabufDesc { + pub width: i32, + pub height: i32, + pub drm_fourcc: u32, + pub modifier: u64, + pub planes: Vec, + buffer_handle: BufferHandle, + requeue: std::sync::Arc>>, +} + +impl std::fmt::Debug for DmabufDesc { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DmabufDesc") + .field("width", &self.width) + .field("height", &self.height) + .field("drm_fourcc", &self.drm_fourcc) + .field("modifier", &self.modifier) + .field("planes", &self.planes) + .finish_non_exhaustive() + } +} + +impl Drop for DmabufDesc { + fn drop(&mut self) { + // Return the held PipeWire buffer once the import that read it is done + // (which is why this runs at drop, after `Capture::stage`). Pushed to the + // queue rather than re-queued here because re-queue must run on the main + // loop, not the PipeWire thread that supersedes a frame — see + // FrameMailbox and osc_pw_requeue_buffer. + if !self.buffer_handle.ptr.is_null() { + if let Ok(mut queue) = self.requeue.lock() { + queue.push(self.buffer_handle); + } + } + } +} + +/// One dmabuf plane: an OWNED (dup'd) fd plus its layout. The dup is taken when the +/// frame is claimed and closed when the owning `DmabufDesc` drops. Owning it — not +/// borrowing the PipeWire buffer's fd — is what keeps the plane valid if a +/// renegotiation destroys the buffer set (which closes the original fds and reuses +/// the numbers) while this plane is still queued for import. +#[derive(Debug)] +pub struct DmabufPlane { + pub fd: OwnedFd, + pub offset: i32, + pub stride: i32, } /// A rectangle inside a captured frame, in stream pixels. @@ -207,6 +304,11 @@ pub struct FrameMailbox { inner: std::sync::Mutex, received: std::sync::atomic::AtomicU64, dropped: std::sync::atomic::AtomicU64, + /// PipeWire buffers held for a dmabuf import, to be re-queued once their + /// `DmabufDesc` drops (import done, or frame superseded). Drained by the main + /// loop, which re-queues each through the shim's locked path. Shared into each + /// `DmabufDesc` so its Drop can push here from either thread. + requeue: std::sync::Arc>>, } #[derive(Debug, Default)] @@ -257,6 +359,43 @@ impl FrameMailbox { height: meta.crop_height, }, has_crop: meta.has_crop != 0, + dmabuf: None, + }); + self.received.fetch_add(1, Ordering::Relaxed); + } + + /// Stores a tiled dmabuf frame — the descriptor only, no pixel copy. Same + /// newest-wins discipline as [`Self::put`]; a superseded frame's owned fds + /// close when its `Frame` drops here. + fn put_dmabuf(&self, desc: DmabufDesc, meta: &RawFrame) { + use std::sync::atomic::Ordering; + + let Ok(mut inner) = self.inner.lock() else { + self.dropped.fetch_add(1, Ordering::Relaxed); + return; + }; + let pixels = match inner.pending.take() { + Some(stale) => { + self.dropped.fetch_add(1, Ordering::Relaxed); + stale.pixels + } + None => inner.spare.take().unwrap_or_default(), + }; + inner.pending = Some(Frame { + pixels, + stride: meta.stride as usize, + width: meta.width, + height: meta.height, + video_format: meta.video_format, + pts_ns: meta.pts_ns, + crop: CropRect { + x: meta.crop_x, + y: meta.crop_y, + width: meta.crop_width, + height: meta.crop_height, + }, + has_crop: meta.has_crop != 0, + dmabuf: Some(desc), }); self.received.fetch_add(1, Ordering::Relaxed); } @@ -276,6 +415,21 @@ impl FrameMailbox { inner.spare = Some(pixels); } + /// A clone of the held-buffer re-queue queue, for a `DmabufDesc` to push its + /// PipeWire buffer to when it drops. + fn requeue_queue(&self) -> std::sync::Arc>> { + self.requeue.clone() + } + + /// Takes the PipeWire buffers whose dmabuf imports have completed (or were + /// superseded), for the main loop to re-queue through the shim. + pub fn drain_requeue(&self) -> Vec { + match self.requeue.lock() { + Ok(mut queue) => std::mem::take(&mut *queue), + Err(_) => Vec::new(), + } + } + /// Frames the compositor delivered. pub fn received(&self) -> u64 { self.received.load(std::sync::atomic::Ordering::Relaxed) @@ -755,8 +909,12 @@ impl Session { node_id: u32, sink: Sink, frames: Option>, + // Offer dmabuf before shm for a whole-monitor GPU import (issue #507). + // Only honoured for a video session; ignored for cursor-only. + prefer_dmabuf: bool, ) -> Result { let want_video = i32::from(frames.is_some()); + let prefer_dmabuf = i32::from(want_video != 0 && prefer_dmabuf); let state = Box::new(CallbackState { sink, frames }); let user = &*state as *const CallbackState as *mut c_void; let callbacks = RawCallbacks { @@ -776,6 +934,7 @@ impl Session { fd.into_raw_fd(), node_id, want_video, + prefer_dmabuf, &callbacks, err.as_mut_ptr(), ERR_LEN, @@ -787,6 +946,16 @@ impl Session { Ok(Self { raw, _state: state }) } + + /// Re-queues a PipeWire buffer a dmabuf frame took ownership of, once its + /// import has copied the pixels. Call from the main loop (NOT the PipeWire + /// thread) — the shim takes the thread-loop lock. Drain the mailbox's + /// `drain_requeue` for the handles. + pub fn requeue(&self, handle: BufferHandle) { + // SAFETY: `raw` is a live session for the lifetime of `self`; the handle + // is an opaque pw_buffer token the shim validates and only re-queues. + unsafe { osc_pw_requeue_buffer(self.raw, handle.ptr, handle.generation) }; + } } impl Drop for Session { @@ -833,35 +1002,114 @@ extern "C" fn on_format(user: *mut c_void, format: *const RawFormat) { }); } -extern "C" fn on_frame(user: *mut c_void, frame: *const RawFrame) { - with_state(user, |state| { - let Some(mailbox) = state.frames.as_ref() else { - return; - }; - if frame.is_null() { - return; +/// Returns 1 when we TAKE OWNERSHIP of the PipeWire buffer — a tiled dmabuf held +/// out of the queue until the main loop imports it — so the shim must not re-queue +/// it. 0 otherwise (the CPU path, or any frame we decline), which re-queues as +/// before. +extern "C" fn on_frame(user: *mut c_void, frame: *const RawFrame) -> i32 { + if user.is_null() || frame.is_null() { + return 0; + } + // SAFETY: `user` is the CallbackState pointer given to osc_pw_start, valid for + // the session's lifetime; `frame` is valid for the callback's duration. + let state = unsafe { &*(user as *const CallbackState) }; + // Same guard `with_state` gives every other callback, which this one cannot use + // because it returns a value: a panic (an allocation failure in + // `Vec::with_capacity`, a capacity overflow in `put`) must not unwind across the + // `extern "C"` boundary and abort the helper. Decline the frame on panic (0) so + // the shim re-queues it rather than leaking the buffer. + catch_unwind(AssertUnwindSafe(|| on_frame_inner(state, frame))).unwrap_or(0) +} + +/// The body of [`on_frame`], split out so the callback can wrap it in +/// `catch_unwind`. `frame` is non-null (checked by the caller) and valid for the +/// callback's duration. +fn on_frame_inner(state: &CallbackState, frame: *const RawFrame) -> i32 { + let Some(mailbox) = state.frames.as_ref() else { + return 0; + }; + // SAFETY: non-null (checked in `on_frame`) and valid for the callback duration. + let frame = unsafe { &*frame }; + + // Tiled dmabuf: no pixels to copy. Take the PipeWire buffer (hold it out of + // the queue) so the plane fds AND their content stay valid until the main-loop + // import copies the surface; the buffer is re-queued when the DmabufDesc drops. + if frame.is_dmabuf != 0 { + // Decline a buffer the C side could not register (its live-buffer table was + // full — two overlapping sets across a renegotiation). Its generation is 0, + // which `osc_pw_requeue_buffer` refuses to re-queue, so holding it would + // leak it out of the pool for good. Returning 0 lets the shim re-queue it + // now; the frame is dropped instead (the previous one is held forward). + if frame.buffer_generation == 0 { + return 0; } - // SAFETY: non-NULL for the duration of the callback, by contract. - let frame = unsafe { &*frame }; - if frame.data.is_null() || frame.stride <= 0 || frame.height <= 0 { - return; + let n = frame.n_planes.clamp(0, 4) as usize; + if n == 0 { + return 0; } - // Copy only the rows, not the whole mapping. `size` can include trailing - // slack the compositor allocated, and re-checking the product here means - // the slice below cannot outrun the region the C side validated. - let Some(rows) = (frame.stride as usize).checked_mul(frame.height as usize) else { - return; - }; - if rows > frame.size { - return; + let mut planes = Vec::with_capacity(n); + for i in 0..n { + let fd = frame.plane_fd[i]; + if fd < 0 { + return 0; + } + // Dup the plane fd so the descriptor owns a handle independent of the + // PipeWire buffer's lifetime: if a renegotiation destroys the buffer set + // while this desc is still queued for import, the original fds are closed + // and their numbers reused, and a borrowed fd would then import an + // unrelated buffer. The dup keeps the dmabuf alive until the OwnedFd drops + // with the desc, after `Capture::stage`; VAAPI dups again during surface + // creation, so it costs nothing past import. + // SAFETY: `fd` is valid for this callback; `try_clone_to_owned` dups it. + let Ok(owned) = (unsafe { BorrowedFd::borrow_raw(fd) }).try_clone_to_owned() else { + // fd exhaustion: decline. `planes` drops here, closing the dups taken + // so far, and the shim re-queues the buffer. + return 0; + }; + planes.push(DmabufPlane { + fd: owned, + offset: frame.plane_offset[i], + stride: frame.plane_stride[i], + }); } - // SAFETY: the shim clamped `size` against the mapping's `maxsize` before - // the callback, `rows <= size` was just checked, and the mapping stays - // live until this returns. - let pixels = unsafe { std::slice::from_raw_parts(frame.data, rows) }; - mailbox.put(pixels, frame); + mailbox.put_dmabuf( + DmabufDesc { + width: frame.width, + height: frame.height, + drm_fourcc: frame.drm_fourcc, + modifier: frame.modifier, + planes, + buffer_handle: BufferHandle { + ptr: frame.buffer_handle, + generation: frame.buffer_generation, + }, + requeue: mailbox.requeue_queue(), + }, + frame, + ); (state.sink)(StreamEvent::FrameReady); - }); + return 1; + } + + if frame.data.is_null() || frame.stride <= 0 || frame.height <= 0 { + return 0; + } + // Copy only the rows, not the whole mapping. `size` can include trailing + // slack the compositor allocated, and re-checking the product here means + // the slice below cannot outrun the region the C side validated. + let Some(rows) = (frame.stride as usize).checked_mul(frame.height as usize) else { + return 0; + }; + if rows > frame.size { + return 0; + } + // SAFETY: the shim clamped `size` against the mapping's `maxsize` before + // the callback, `rows <= size` was just checked, and the mapping stays + // live until this returns. + let pixels = unsafe { std::slice::from_raw_parts(frame.data, rows) }; + mailbox.put(pixels, frame); + (state.sink)(StreamEvent::FrameReady); + 0 } extern "C" fn on_buffer_info( @@ -1028,7 +1276,8 @@ mod tests { // The advertised modifier set is a real set, not a wildcard: a tiled or // compressed buffer cannot be read through a plain mmap, so it must fail // negotiation rather than be accepted and decoded into garbage. - // 0x0300000000000001 = a vendor (AMD) modifier, neither LINEAR nor INVALID. + // 0x0300000000000001 = a vendor (NVIDIA — modifier vendor byte 0x03) modifier, + // neither LINEAR nor INVALID. assert_eq!( enum_format_accepts_dmabuf_producer(true, 0x0300_0000_0000_0001), 0, @@ -1086,6 +1335,8 @@ mod tests { // Cursor-only: this test is about negotiation reaching `streaming` // and about which metadata survives, neither of which needs pixels. None, + // Cursor-only, so dmabuf preference is irrelevant. + false, ) .expect("stream must connect"); diff --git a/electron/native/screencapturekit/Package.swift b/electron/native/screencapturekit/Package.swift index b865f8ae6..e478693b1 100644 --- a/electron/native/screencapturekit/Package.swift +++ b/electron/native/screencapturekit/Package.swift @@ -4,6 +4,24 @@ import PackageDescription let package = Package( name: "OpenScreenScreenCaptureKitHelper", + // macOS 13 is DELIBERATE, and it is the same number the app declares in + // electron-builder.json5 (`mac.minimumSystemVersion`) and promises in the README. + // Those three must move together; scripts/check-macos-deployment-target.test.mjs + // asserts this one never rises above what the app declares. + // + // It has to be at least 13 regardless: ScreenCaptureRecorder is + // `@available(macOS 13.0, *)` and its main() hard-guards `#available(macOS 13.0, *)`, + // because SCStream's usable surface starts there. + // + // What this block is NOT allowed to become is higher than the declared floor, which is + // how #515 happened. The floor was set here when ScreenCaptureKit was the only target; + // openscreen-macos-cursor-helper was added later and inherited it, because SwiftPM has + // no per-target override. The app then advertised macOS 12 while shipping a 13-only + // helper, and the damage was not the version number: at a deployment target >= 13 the + // linker resolves the Swift Foundation overlay symbols against Foundation.framework and + // drops /usr/lib/swift/libswiftFoundation.dylib from the load commands, so on macOS 12 + // the helper died in dyld before it could speak — which the app reported to the user as + // a denied Accessibility grant. platforms: [ .macOS(.v13) ], diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 75163dee7..5add8074d 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -413,6 +413,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { let size = captureSize( for: filter, fallbackPointSize: window.frame.size, + // Unrelated to `initializeCoreGraphicsWindowServerConnection()`: that call + // exists purely for its side effect at process startup, this one wants the + // actual display ID as a fallback when no display intersects the window. fallbackDisplayId: candidateDisplay?.displayID ?? CGMainDisplayID() ) return CaptureTarget( @@ -800,8 +803,21 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { @main struct OpenScreenScreenCaptureKitHelper { + // This helper is a plain command-line executable, so nothing has connected it to the + // window server yet. `SCContentFilter(desktopIndependentWindow:)` reaches into SkyLight + // (`SLSGetDisplaysWithRect`) to find the display a window sits on, and SkyLight aborts + // with `CGS_REQUIRE_INIT` when CoreGraphics was never initialised in the process — so + // every window capture crashed before it produced a frame, while display capture (which + // never resolves a rect) worked fine. Touching any CoreGraphics display API first + // performs that initialisation. + private static func initializeCoreGraphicsWindowServerConnection() { + _ = CGMainDisplayID() + } + static func main() async { do { + initializeCoreGraphicsWindowServerConnection() + guard CommandLine.arguments.count == 2 else { throw HelperError.invalidArguments } diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index c2947df77..99041a02b 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -89,3 +89,25 @@ target_link_libraries(cursor-sampler PRIVATE gdi32 gdiplus ) + +add_executable(audio_sample_utils_test + src/audio_sample_utils.cpp + src/audio_sample_utils.h + src/audio_sample_utils_test.cpp +) + +target_compile_definitions(audio_sample_utils_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(audio_sample_utils_test PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(audio_sample_utils_test PRIVATE + mf + mfplat + mfreadwrite + mfuuid + ole32 +) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp index 5e60860c9..96847ee7d 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,21 @@ double readMappedChannel(const BYTE* source, const AudioInputFormat& format, siz return readSampleAsDouble(source, format, frameIndex, std::min(targetChannel, format.channels - 1)); } +UINT32 aacCompatibleSampleRate(UINT32 sampleRate) { + constexpr UINT32 kAacSampleRates[] = { + 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000, + }; + if (sampleRate == 0) { + return 48000; + } + for (UINT32 rate : kAacSampleRates) { + if (sampleRate == rate) { + return rate; + } + } + return 48000; +} + } // namespace constexpr int64_t HnsPerSecond = 10'000'000; @@ -100,10 +116,16 @@ bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputForm left.avgBytesPerSec == right.avgBytesPerSec; } +// Microsoft AAC encoder (MFAudioFormat_AAC) sample rates. WASAPI loopback +// often reports 96000 or 192000; those are legal PCM mix rates but not AAC +// input rates, and SetInputMediaType then fails with MF_E_INVALIDMEDIATYPE +// (0xc00d36b4). Keep legal rates as-is so a working 44100/48000 path is +// unchanged; snap everything else (including 0) to 48000. The mixer already +// resamples through convertAudioWithGain when the source rate differs. AudioInputFormat makeAacCompatibleAudioFormat(const AudioInputFormat& source) { AudioInputFormat format{}; format.subtype = MFAudioFormat_PCM; - format.sampleRate = source.sampleRate > 0 ? source.sampleRate : 48000; + format.sampleRate = aacCompatibleSampleRate(source.sampleRate); format.channels = 2; format.bitsPerSample = 16; format.blockAlign = format.channels * (format.bitsPerSample / 8); @@ -168,6 +190,19 @@ void convertAudioWithGain( const AudioInputFormat& targetFormat, double gain, std::vector& destination) { + std::vector discardedRemainder; + convertAudioWithGain( + source, byteCount, sourceFormat, targetFormat, gain, destination, discardedRemainder); +} + +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination, + std::vector& remainder) { if (!source || byteCount == 0 || sourceFormat.blockAlign == 0 || targetFormat.blockAlign == 0 || sourceFormat.sampleRate == 0 || targetFormat.sampleRate == 0 || sourceFormat.channels == 0 || targetFormat.channels == 0) { @@ -180,12 +215,65 @@ void convertAudioWithGain( return; } - const size_t sourceFrames = byteCount / sourceFormat.blockAlign; - if (sourceFrames == 0) { + const size_t packetFrames = byteCount / sourceFormat.blockAlign; + if (packetFrames == 0) { destination.clear(); return; } + // Integer-factor downsample (96 kHz / 192 kHz -> 48 kHz): average each + // group of source frames instead of picking one. Nearest-neighbour + // decimation aliases content above the new Nyquist into the recording. + // Incomplete groups stay in remainder so the next packet can finish them. + if (sourceFormat.sampleRate > targetFormat.sampleRate && + sourceFormat.sampleRate % targetFormat.sampleRate == 0) { + const UINT32 factor = sourceFormat.sampleRate / targetFormat.sampleRate; + if (remainder.size() % sourceFormat.blockAlign != 0) { + remainder.clear(); + } + std::vector combined; + combined.reserve(remainder.size() + byteCount); + combined.insert(combined.end(), remainder.begin(), remainder.end()); + combined.insert(combined.end(), source, source + byteCount); + const size_t totalFrames = combined.size() / sourceFormat.blockAlign; + const size_t targetFrames = totalFrames / factor; + const size_t consumedFrames = targetFrames * factor; + const size_t leftoverBytes = (totalFrames - consumedFrames) * sourceFormat.blockAlign; + if (targetFrames == 0) { + destination.clear(); + remainder.swap(combined); + return; + } + destination.assign(targetFrames * targetFormat.blockAlign, 0); + for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { + for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { + double sum = 0.0; + for (UINT32 tap = 0; tap < factor; ++tap) { + sum += readMappedChannel( + combined.data(), + sourceFormat, + targetFrame * factor + tap, + channel, + targetFormat.channels); + } + writeSampleFromDouble( + destination.data(), + targetFormat, + targetFrame, + channel, + (sum / static_cast(factor)) * gain); + } + } + remainder.assign( + combined.begin() + static_cast(consumedFrames * sourceFormat.blockAlign), + combined.end()); + if (remainder.size() != leftoverBytes) { + remainder.resize(leftoverBytes); + } + return; + } + + const size_t sourceFrames = packetFrames; const double rateRatio = static_cast(targetFormat.sampleRate) / static_cast(sourceFormat.sampleRate); const size_t targetFrames = std::max(1, static_cast(std::llround(sourceFrames * rateRatio))); @@ -193,17 +281,20 @@ void convertAudioWithGain( for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { const double sourcePosition = static_cast(targetFrame) / rateRatio; - const size_t sourceFrame = std::min( - sourceFrames - 1, - static_cast(std::llround(sourcePosition))); + const size_t sourceFrame = std::min(sourceFrames - 1, static_cast(sourcePosition)); + const size_t nextFrame = std::min(sourceFrames - 1, sourceFrame + 1); + const double frac = sourcePosition - static_cast(sourceFrame); for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { - const double sample = readMappedChannel( - source, - sourceFormat, - sourceFrame, + const double a = readMappedChannel( + source, sourceFormat, sourceFrame, channel, targetFormat.channels); + const double b = readMappedChannel( + source, sourceFormat, nextFrame, channel, targetFormat.channels); + writeSampleFromDouble( + destination.data(), + targetFormat, + targetFrame, channel, - targetFormat.channels); - writeSampleFromDouble(destination.data(), targetFormat, targetFrame, channel, sample * gain); + (a + (b - a) * frac) * gain); } } } @@ -280,6 +371,8 @@ bool AudioMixer::start() { emittedFrames_ = 0; timelineStarted_ = false; paused_ = false; + systemResampleRemainder_.clear(); + microphoneResampleRemainder_.clear(); thread_ = std::thread([this] { mixLoop(); }); @@ -291,6 +384,8 @@ void AudioMixer::beginTimeline() { std::scoped_lock lock(mutex_); systemQueue_.clear(); microphoneQueue_.clear(); + systemResampleRemainder_.clear(); + microphoneResampleRemainder_.clear(); emittedFrames_ = 0; timelineStarted_ = true; } @@ -304,6 +399,8 @@ void AudioMixer::setPaused(bool paused) { if (paused_) { systemQueue_.clear(); microphoneQueue_.clear(); + systemResampleRemainder_.clear(); + microphoneResampleRemainder_.clear(); } } cv_.notify_all(); @@ -327,7 +424,7 @@ void AudioMixer::pushSystem(const BYTE* data, DWORD byteCount) { if (paused_) { return; } - append(systemQueue_, data, byteCount, systemFormat_, 1.0); + append(systemQueue_, data, byteCount, systemFormat_, 1.0, systemResampleRemainder_); } cv_.notify_all(); } @@ -342,7 +439,13 @@ void AudioMixer::pushMicrophone(const BYTE* data, DWORD byteCount) { if (paused_) { return; } - append(microphoneQueue_, data, byteCount, microphoneFormat_, microphoneGain_); + append( + microphoneQueue_, + data, + byteCount, + microphoneFormat_, + microphoneGain_, + microphoneResampleRemainder_); } cv_.notify_all(); } @@ -352,12 +455,13 @@ void AudioMixer::append( const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, - double gain) { + double gain, + std::vector& remainder) { if (!data || byteCount == 0) { return; } - convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_); + convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_, remainder); queue.insert(queue.end(), gainBuffer_.begin(), gainBuffer_.end()); } diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h index 0bdbc0809..0f8e6b69f 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.h +++ b/electron/native/wgc-capture/src/audio_sample_utils.h @@ -27,6 +27,14 @@ void convertAudioWithGain( const AudioInputFormat& targetFormat, double gain, std::vector& destination); +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination, + std::vector& remainder); void mixAudioInPlace( std::vector& destination, const BYTE* source, @@ -63,7 +71,8 @@ class AudioMixer { const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, - double gain); + double gain, + std::vector& remainder); bool pop(std::vector& queue, std::vector& chunk, size_t byteCount); void mixLoop(); @@ -78,6 +87,8 @@ class AudioMixer { std::condition_variable cv_; std::vector systemQueue_; std::vector microphoneQueue_; + std::vector systemResampleRemainder_; + std::vector microphoneResampleRemainder_; std::vector gainBuffer_; std::thread thread_; std::atomic stopRequested_ = false; diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp new file mode 100644 index 000000000..8b74b9144 --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp @@ -0,0 +1,412 @@ +#include "audio_sample_utils.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_ran = 0; +int g_failed = 0; + +AudioInputFormat makeFormat( + GUID subtype, + UINT32 sampleRate, + UINT32 channels, + UINT32 bitsPerSample) { + AudioInputFormat format{}; + format.subtype = subtype; + format.sampleRate = sampleRate; + format.channels = channels; + format.bitsPerSample = bitsPerSample; + format.blockAlign = channels * (bitsPerSample / 8); + format.avgBytesPerSec = sampleRate * format.blockAlign; + return format; +} + +void expect(const char* name, bool ok, const std::string& detail) { + g_ran += 1; + if (ok) { + std::cout << "PASS " << name << "\n"; + return; + } + g_failed += 1; + std::cout << "FAIL " << name << " " << detail << "\n"; +} + +void skip(const char* name, const std::string& reason) { + std::cout << "SKIP " << name << " " << reason << "\n"; +} + +struct TempMp4 { + std::wstring path; + explicit TempMp4(std::wstring p) : path(std::move(p)) { + DeleteFileW(path.c_str()); + } + ~TempMp4() { + DeleteFileW(path.c_str()); + } + TempMp4(const TempMp4&) = delete; + TempMp4& operator=(const TempMp4&) = delete; +}; + +std::string describe(const AudioInputFormat& format) { + return "sampleRate=" + std::to_string(format.sampleRate) + + " channels=" + std::to_string(format.channels) + + " bits=" + std::to_string(format.bitsPerSample); +} + +std::wstring tempMp4Path() { + wchar_t dir[MAX_PATH]{}; + GetTempPathW(MAX_PATH, dir); + return std::wstring(dir) + L"openscreen-mf-aac-probe-" + + std::to_wstring(GetCurrentProcessId()) + L".mp4"; +} + +HRESULT trySetAacPcmRate(UINT32 sampleRate, IMFAttributes* attributes = nullptr) { + TempMp4 tmp(tempMp4Path()); + + Microsoft::WRL::ComPtr writer; + HRESULT hr = MFCreateSinkWriterFromURL(tmp.path.c_str(), nullptr, attributes, &writer); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr outputType; + hr = MFCreateMediaType(&outputType); + if (FAILED(hr)) { + return hr; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + outputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + outputType->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + outputType->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + outputType->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + outputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, 24000); + outputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + DWORD streamIndex = 0; + hr = writer->AddStream(outputType.Get(), &streamIndex); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr inputType; + hr = MFCreateMediaType(&inputType); + if (FAILED(hr)) { + return hr; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + inputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); + inputType->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + inputType->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + inputType->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + inputType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, 4); + inputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, sampleRate * 4); + inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + + hr = writer->SetInputMediaType(streamIndex, inputType.Get(), nullptr); + writer.Reset(); + return hr; +} + +// Same rates as trySetAacPcmRate, but with an H.264 stream first — the helper's +// topology. Distinguishes "96 kHz AAC is illegal" from "audio-only MP4 sink +// writer refuses this type". +HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { + TempMp4 tmp(tempMp4Path()); + + Microsoft::WRL::ComPtr writer; + HRESULT hr = MFCreateSinkWriterFromURL(tmp.path.c_str(), nullptr, nullptr, &writer); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr videoOut; + hr = MFCreateMediaType(&videoOut); + if (FAILED(hr)) { + return hr; + } + videoOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + videoOut->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + videoOut->SetUINT32(MF_MT_AVG_BITRATE, 1'000'000); + videoOut->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + MFSetAttributeSize(videoOut.Get(), MF_MT_FRAME_SIZE, 320, 240); + MFSetAttributeRatio(videoOut.Get(), MF_MT_FRAME_RATE, 30, 1); + MFSetAttributeRatio(videoOut.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + + DWORD videoIndex = 0; + hr = writer->AddStream(videoOut.Get(), &videoIndex); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr audioOut; + hr = MFCreateMediaType(&audioOut); + if (FAILED(hr)) { + return hr; + } + audioOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + audioOut->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + audioOut->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + audioOut->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + audioOut->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + audioOut->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, 24000); + audioOut->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + DWORD audioIndex = 0; + hr = writer->AddStream(audioOut.Get(), &audioIndex); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr videoIn; + hr = MFCreateMediaType(&videoIn); + if (FAILED(hr)) { + return hr; + } + videoIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + videoIn->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + videoIn->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + videoIn->SetUINT32(MF_MT_DEFAULT_STRIDE, 320 * 4); + MFSetAttributeSize(videoIn.Get(), MF_MT_FRAME_SIZE, 320, 240); + MFSetAttributeRatio(videoIn.Get(), MF_MT_FRAME_RATE, 30, 1); + MFSetAttributeRatio(videoIn.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + hr = writer->SetInputMediaType(videoIndex, videoIn.Get(), nullptr); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr audioIn; + hr = MFCreateMediaType(&audioIn); + if (FAILED(hr)) { + return hr; + } + audioIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + audioIn->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); + audioIn->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + audioIn->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + audioIn->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + audioIn->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, 4); + audioIn->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, sampleRate * 4); + audioIn->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + hr = writer->SetInputMediaType(audioIndex, audioIn.Get(), nullptr); + writer.Reset(); + return hr; +} + +} // namespace + +int main() { + const AudioInputFormat diagnostic = makeFormat(MFAudioFormat_Float, 96000, 8, 32); + const AudioInputFormat snapped = makeAacCompatibleAudioFormat(diagnostic); + expect( + "diag-96000-8ch", + snapped.sampleRate == 48000 && snapped.channels == 2 && snapped.bitsPerSample == 16 && + snapped.subtype == MFAudioFormat_PCM, + describe(snapped)); + + const AudioInputFormat keep48000 = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 48000, 2, 16)); + expect("keep-48000", keep48000.sampleRate == 48000, describe(keep48000)); + + const AudioInputFormat keep44100 = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 44100, 2, 16)); + expect("keep-44100", keep44100.sampleRate == 44100, describe(keep44100)); + + const AudioInputFormat zeroRate = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 0, 2, 16)); + expect("zero-rate", zeroRate.sampleRate == 48000, describe(zeroRate)); + + const AudioInputFormat keep32000 = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 32000, 2, 16)); + expect("keep-32000", keep32000.sampleRate == 32000, describe(keep32000)); + + const AudioInputFormat source96k = makeFormat(MFAudioFormat_PCM, 96000, 2, 16); + const AudioInputFormat target48k = makeAacCompatibleAudioFormat(source96k); + const UINT32 sourceFrames = 96000; + std::vector source(static_cast(sourceFrames) * source96k.blockAlign, 0); + auto* samples = reinterpret_cast(source.data()); + for (UINT32 frame = 0; frame < sourceFrames; frame += 1) { + samples[frame * 2] = static_cast(frame % 32767); + samples[frame * 2 + 1] = static_cast((frame * 3) % 32767); + } + std::vector converted; + convertAudioWithGain( + source.data(), + static_cast(source.size()), + source96k, + target48k, + 1.0, + converted); + const size_t convertedFrames = + target48k.blockAlign == 0 ? 0 : converted.size() / target48k.blockAlign; + const bool frameCountOk = + convertedFrames == 48000 || convertedFrames == 47999 || convertedFrames == 48001; + expect( + "resample-frame-count", + target48k.sampleRate == 48000 && frameCountOk, + "frames=" + std::to_string(convertedFrames) + " " + describe(target48k)); + + // 96 kHz Nyquist square (+/- full scale) must not survive 2:1 as a tone. + std::vector nyquist(8 * source96k.blockAlign, 0); + auto* nyquistSamples = reinterpret_cast(nyquist.data()); + for (size_t frame = 0; frame < 8; frame += 1) { + const int16_t v = (frame % 2 == 0) ? 32767 : -32767; + nyquistSamples[frame * 2] = v; + nyquistSamples[frame * 2 + 1] = v; + } + std::vector nyquistOut; + convertAudioWithGain(nyquist.data(), static_cast(nyquist.size()), source96k, target48k, 1.0, nyquistOut); + const auto* down = reinterpret_cast(nyquistOut.data()); + const size_t downFrames = nyquistOut.size() / target48k.blockAlign; + bool folded = downFrames == 4; + for (size_t i = 0; folded && i < downFrames * 2; i += 1) { + folded = std::abs(static_cast(down[i])) <= 1; + } + expect("resample-96k-nyquist-box", folded, "frames=" + std::to_string(downFrames)); + + auto fillStereoFrame = [](std::vector& packet, int16_t left, int16_t right) { + auto* samples = reinterpret_cast(packet.data()); + samples[0] = left; + samples[1] = right; + }; + const auto destFrames = [&](const std::vector& out) -> size_t { + return target48k.blockAlign == 0 ? 0 : out.size() / target48k.blockAlign; + }; + const auto remainderFrames = [&](const std::vector& rem) -> size_t { + return source96k.blockAlign == 0 ? 0 : rem.size() / source96k.blockAlign; + }; + + std::vector remainder; + std::vector shortPkt(source96k.blockAlign, 0); + fillStereoFrame(shortPkt, 12345, -12345); + std::vector shortOut; + convertAudioWithGain( + shortPkt.data(), + static_cast(shortPkt.size()), + source96k, + target48k, + 1.0, + shortOut, + remainder); + expect( + "resample-96k-short-packet", + shortOut.empty() && remainderFrames(remainder) == 1, + "dest=" + std::to_string(shortOut.size()) + " rem=" + std::to_string(remainder.size())); + + remainder.clear(); + std::vector oneA(source96k.blockAlign, 0); + std::vector oneB(source96k.blockAlign, 0); + fillStereoFrame(oneA, 1000, 2000); + fillStereoFrame(oneB, 3000, 4000); + std::vector outA; + std::vector outB; + convertAudioWithGain(oneA.data(), static_cast(oneA.size()), source96k, target48k, 1.0, outA, remainder); + convertAudioWithGain(oneB.data(), static_cast(oneB.size()), source96k, target48k, 1.0, outB, remainder); + expect( + "resample-96k-one-frame-packets", + destFrames(outA) == 0 && destFrames(outB) == 1 && remainder.empty(), + "a=" + std::to_string(destFrames(outA)) + " b=" + std::to_string(destFrames(outB)) + + " rem=" + std::to_string(remainderFrames(remainder))); + + remainder.clear(); + std::vector threePkt(3 * source96k.blockAlign, 0); + std::vector onePkt(source96k.blockAlign, 0); + fillStereoFrame(onePkt, 5000, 6000); + std::vector threeOut; + std::vector oneOut; + convertAudioWithGain( + threePkt.data(), static_cast(threePkt.size()), source96k, target48k, 1.0, threeOut, remainder); + convertAudioWithGain( + onePkt.data(), static_cast(onePkt.size()), source96k, target48k, 1.0, oneOut, remainder); + expect( + "resample-96k-remainder-three-then-one", + destFrames(threeOut) == 1 && destFrames(oneOut) == 1 && remainder.empty(), + "three=" + std::to_string(destFrames(threeOut)) + " one=" + std::to_string(destFrames(oneOut)) + + " rem=" + std::to_string(remainderFrames(remainder))); + + remainder.clear(); + std::vector remainderFullOut; + convertAudioWithGain( + source.data(), + static_cast(source.size()), + source96k, + target48k, + 1.0, + remainderFullOut, + remainder); + const size_t remainderFullFrames = destFrames(remainderFullOut); + const bool remainderFullOk = + remainderFullFrames == 48000 || remainderFullFrames == 47999 || remainderFullFrames == 48001; + expect( + "resample-96k-remainder-full", + remainderFullOk && remainder.empty(), + "frames=" + std::to_string(remainderFullFrames) + " rem=" + std::to_string(remainder.size())); + + HRESULT mfHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (FAILED(mfHr) && mfHr != RPC_E_CHANGED_MODE) { + skip("mf-startup", "CoInitializeEx failed — no Media Foundation on this host"); + } else { + mfHr = MFStartup(MF_VERSION); + if (FAILED(mfHr)) { + skip("mf-startup", "MFStartup hr=" + std::to_string(static_cast(mfHr))); + } else { + expect("mf-startup", true, ""); + const auto runReject = [](const char* name, HRESULT hr) { + char hex[16]{}; + sprintf_s(hex, "0x%08lx", static_cast(hr)); + std::cout << "MF_RAW " << name << " hr=" << hex << "\n"; + if (FAILED(hr) && static_cast(hr) == 0xc00d36b4ul) { + expect(name, true, ""); + } else if (SUCCEEDED(hr)) { + skip(name, "host AAC accepts 96 kHz"); + } else { + skip(name, std::string("host cannot probe this rate hr=") + hex); + } + }; + const auto runAccept = [](const char* name, HRESULT hr) { + char hex[16]{}; + sprintf_s(hex, "0x%08lx", static_cast(hr)); + std::cout << "MF_RAW " << name << " hr=" << hex << "\n"; + if (SUCCEEDED(hr)) { + expect(name, true, ""); + return true; + } + skip(name, std::string("host has no AAC encoder hr=") + hex); + return false; + }; + runReject("mf-reject-96000", trySetAacPcmRate(96000)); + if (runAccept("mf-accept-48000", trySetAacPcmRate(48000))) { + runReject("mf-reject-96000-with-video", trySetAacPcmRateWithVideo(96000)); + runAccept("mf-accept-48000-with-video", trySetAacPcmRateWithVideo(48000)); + Microsoft::WRL::ComPtr swAttr; + if (FAILED(MFCreateAttributes(&swAttr, 1))) { + skip("mf-reject-96000-sw-attr", "MFCreateAttributes failed"); + } else { + swAttr->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, FALSE); + runReject("mf-reject-96000-sw-attr", trySetAacPcmRate(96000, swAttr.Get())); + runAccept("mf-accept-48000-sw-attr", trySetAacPcmRate(48000, swAttr.Get())); + } + } + MFShutdown(); + } + } + + std::cout << "ran " << g_ran << " tests\n"; + if (g_failed != 0) { + std::cout << g_failed << " failed\n"; + return 1; + } + return 0; +} diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 1479bc842..614e84d00 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -146,6 +146,15 @@ int readEnvInt(const char* name, int fallback) { } } +// Rollback lever for the pull-based WGC frame delivery (default; see +// wgc_session.h). Forces the previously-shipped FrameArrived-callback path +// instead, for anyone hit by a regression the pull-based path was not tested +// against. Kept only until the pull-based path has enough field time to +// retire this flag and the legacy path with it. +bool useLegacyFrameCallback() { + return readEnvInt("OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK", 0) != 0; +} + std::wstring utf8ToWide(const std::string& value) { if (value.empty()) { return {}; @@ -682,6 +691,22 @@ int main(int argc, char* argv[]) { // ordinary hardware, so the stop path can be regression-tested at all. const int testStallReadbackMs = std::max(0, readEnvInt("OPENSCREEN_WGC_TEST_STALL_READBACK_MS", 0)); + // Test-only: stall the WGC frame *callback* itself while it holds the + // same frame lock, rather than the writer's readback -- the shape + // getopenscreen/openscreen#460 actually reproduced on Intel HD 520 + // ("A WGC frame callback did not finish"). Distinct from + // testStallReadbackMs above because quiesceLegacyCallback()'s drain only + // ever sees the callback side: a stall placed in the writer instead leaves + // callbacksInFlight_ at zero and wgcDrained true, which cannot exercise + // the video-writer-join skip this stall exists to test. + // + // Requires OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK=1: there is no callback + // thread to stall on the default pull path, which is the point of it -- + // the wedge lands on the writer thread instead, where + // testStallReadbackMs already reaches it and the video-writer-join + // watchdog, not the drain, is what bounds it. + const int testStallFrameCallbackMs = + std::max(0, readEnvInt("OPENSCREEN_WGC_TEST_STALL_FRAME_CALLBACK_MS", 0)); std::cout << "{\"event\":\"ready\",\"schemaVersion\":2}" << std::endl; @@ -800,17 +825,43 @@ int main(int argc, char* argv[]) { << jsonEscape(wideToUtf8(microphoneCapture.selectedDeviceName())) << "\""; } std::cout << "}" << std::endl; - encoderAudioFormat = makeAacCompatibleAudioFormat(*audioFormat); + AudioInputFormat sourceForEncoder = *audioFormat; + const int forcedAacSourceRate = readEnvInt("OPENSCREEN_WGC_FORCE_AAC_SOURCE_RATE", 0); + if (forcedAacSourceRate > 0) { + sourceForEncoder.sampleRate = static_cast(forcedAacSourceRate); + sourceForEncoder.avgBytesPerSec = + sourceForEncoder.sampleRate * sourceForEncoder.blockAlign; + } + if (readEnvInt("OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP", 0) == 1) { + encoderAudioFormat = sourceForEncoder; + encoderAudioFormat.subtype = MFAudioFormat_PCM; + encoderAudioFormat.channels = 2; + encoderAudioFormat.bitsPerSample = 16; + encoderAudioFormat.blockAlign = 4; + encoderAudioFormat.avgBytesPerSec = encoderAudioFormat.sampleRate * 4; + } else { + encoderAudioFormat = makeAacCompatibleAudioFormat(sourceForEncoder); + } + std::cout << "{\"event\":\"encoder-audio-format\",\"schemaVersion\":2,\"sampleRate\":" << encoderAudioFormat.sampleRate << ",\"channels\":" << encoderAudioFormat.channels << ",\"bitsPerSample\":" << encoderAudioFormat.bitsPerSample + << ",\"forcedSourceRate\":" << forcedAacSourceRate + << ",\"snapDisabled\":" + << (readEnvInt("OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP", 0) == 1 ? "true" : "false") + << ",\"aacRateProbe\":" + << (readEnvInt("OPENSCREEN_WGC_TEST_INJECT_AAC_RATE_PROBE", 0) == 1 ? "true" + : "false") << "}" << std::endl; } MFEncoderOptions encoderOptions{}; encoderOptions.preferSoftwareEncoder = config.preferSoftwareEncoder; encoderOptions.injectDefaultSinkWriterFailureOnce = injectDefaultSinkWriterFailureOnce; + encoderOptions.skipAacRateSnap = readEnvInt("OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP", 0) == 1; + encoderOptions.injectAacRateProbe = + readEnvInt("OPENSCREEN_WGC_TEST_INJECT_AAC_RATE_PROBE", 0) == 1; // OFF by default. The GPU path exists to dodge a Map() that wedges inside // the display driver on the machine in #252, and it demonstrably fixed // display and window capture there. It also broke recording outright for @@ -865,7 +916,14 @@ int main(int argc, char* argv[]) { << "\",\"container\":\"" << encoder.containerFormat() << "\",\"preferSoftwareEncoder\":" << (config.preferSoftwareEncoder ? "true" : "false") - << "}" << std::endl; + // What BeginWriting() actually landed on, not what the "video" + // field above asked for -- see kVideoEncoderRuntime* in + // mf_encoder.h. "default" plus "software" here means the machine + // never got a hardware encoder in the first place, which is a + // different bug report than "default" plus "hardware" stalling + // on stop. + << ",\"videoEncoderRuntime\":\"" << encoder.videoEncoderRuntime() + << "\"}" << std::endl; MFEncoder webcamEncoder; if (writeSeparateWebcam) { MFEncoderOptions webcamEncoderOptions = encoderOptions; @@ -888,7 +946,20 @@ int main(int argc, char* argv[]) { } } - std::mutex mutex; + // By default, no mutex guards frame handoff: writeVideoFrames is the + // only thread that ever touches WGC or latestFrameTexture. It pulls each + // frame with session.tryGetNextFrame() itself (see wgc_session.h for + // why) instead of a separate thread pushing into a shared, lock-guarded + // texture. A CopyResource that wedges inside the display + // driver (issue #252, and the DXGI path in PR #305 did not avoid it + // either) then blocks only this thread, which is already the thread + // whose job is to notice stopRequested and give up -- there is no second + // thread left for it to take down with it. + // + // OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK=1 reverts to the previously + // shipped push-based design (frameMutex/frameCv guard the handoff from + // WGC's own callback thread) as a rollback lever -- see wgc_session.h. + const bool legacyFrameCallback = useLegacyFrameCallback(); CaptureControl control; std::atomic firstFrameWritten = false; std::atomic encodeFailed = false; @@ -897,39 +968,58 @@ int main(int argc, char* argv[]) { // them is the next bug report, and neither is worth a log line each. std::atomic contendedFrames = 0; Microsoft::WRL::ComPtr latestFrameTexture; - int64_t latestFrameTimestampHns = 0; - int64_t firstFrameTimestampHns = -1; std::vector latestWebcamFrame; int latestWebcamWidth = 0; int latestWebcamHeight = 0; uint64_t latestWebcamSequence = 0; bool hasVisibleWebcamFrame = false; - session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { - if (control.stopRequested || control.paused) { - return; - } - - std::scoped_lock lock(mutex); - if (!latestFrameTexture) { - D3D11_TEXTURE2D_DESC desc{}; - texture->GetDesc(&desc); - desc.BindFlags = 0; - desc.CPUAccessFlags = 0; - desc.MiscFlags = 0; - if (FAILED(session.device()->CreateTexture2D(&desc, nullptr, &latestFrameTexture))) { - encodeFailed = true; - control.requestStop(); + // Legacy-path-only state. frameMutex guards latestFrameTexture/ + // legacyLatestFrameTimestampHns between WGC's callback thread (writer) + // and writeVideoFrames (reader); frameCv wakes the reader. Both are + // unused on the default pull-based path. + std::timed_mutex frameMutex; + std::condition_variable_any frameCv; + int64_t legacyLatestFrameTimestampHns = 0; + + if (legacyFrameCallback) { + session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { + if (control.stopRequested || control.paused) { return; } - } + std::scoped_lock lock(frameMutex); + if (!latestFrameTexture) { + D3D11_TEXTURE2D_DESC desc{}; + texture->GetDesc(&desc); + desc.BindFlags = 0; + desc.CPUAccessFlags = 0; + desc.MiscFlags = 0; + if (FAILED(session.device()->CreateTexture2D(&desc, nullptr, &latestFrameTexture))) { + encodeFailed = true; + control.requestStop(); + return; + } + } - session.context()->CopyResource(latestFrameTexture.Get(), texture); - latestFrameTimestampHns = timestampHns; - if (!firstFrameWritten.exchange(true)) { - control.cv.notify_all(); - } - }); + // Gated on an already-arrived first frame: main() blocks up to 10s + // waiting for firstFrameWritten before it will even print + // recording-started, a startup budget this stall is meant to + // outlast (it needs to still be asleep when `stop` arrives, + // seconds later). Stalling the first frame trips that unrelated + // timeout instead of reaching the steady-state shutdown path this + // exists to test, and does not match the real report either -- + // getopenscreen/openscreen#460's diagnostic shows + // recording-started succeeding before the hang. + if (testStallFrameCallbackMs > 0 && firstFrameWritten.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(testStallFrameCallbackMs)); + } + session.context()->CopyResource(latestFrameTexture.Get(), texture); + legacyLatestFrameTimestampHns = timestampHns; + if (!firstFrameWritten.exchange(true)) { + frameCv.notify_all(); + } + }); + } auto writeVideoFrames = [&]() { const auto frameDuration = std::chrono::duration_cast( @@ -951,6 +1041,8 @@ int main(int argc, char* argv[]) { const int64_t nominalWebcamIntervalHns = static_cast(10'000'000ULL / std::max(1, webcamCapture.fps())); auto nextFrameDue = std::chrono::steady_clock::now(); + int64_t firstFrameTimestampHns = -1; + int64_t latestFrameTimestampHns = 0; while (!control.stopRequested && !encodeFailed) { Microsoft::WRL::ComPtr videoSample; @@ -958,15 +1050,75 @@ int main(int argc, char* argv[]) { bool hasVideoSample = false; bool hasWebcamSample = false; + std::unique_lock legacyLock; { - std::unique_lock lock(mutex); - control.cv.wait_for(lock, std::chrono::milliseconds(100), [&] { - return control.stopRequested.load() || - encodeFailed.load() || - (!control.paused.load() && latestFrameTexture); - }); - if (control.stopRequested || encodeFailed) { - break; + if (legacyFrameCallback) { + // try_lock_for, not a blocking lock: the WGC callback + // holds frameMutex across CopyResource, which can wedge + // inside the display driver and never return (#252). + // This is the exact failure OPENSCREEN_WGC_LEGACY_FRAME_ + // CALLBACK=1 opts back into; a blocking acquire here + // would let it also stall this thread's stop detection. + legacyLock = std::unique_lock(frameMutex, std::defer_lock); + if (!legacyLock.try_lock_for(std::chrono::milliseconds(100))) { + if (control.stopRequested || encodeFailed) { + break; + } + continue; + } + frameCv.wait_for(legacyLock, std::chrono::milliseconds(100), [&] { + return control.stopRequested.load() || + encodeFailed.load() || + (!control.paused.load() && latestFrameTexture); + }); + if (control.stopRequested || encodeFailed) { + break; + } + if (!latestFrameTexture) { + continue; + } + latestFrameTimestampHns = legacyLatestFrameTimestampHns; + } else { + if (control.paused) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + continue; + } + + ID3D11Texture2D* wgcTexture = nullptr; + int64_t wgcTimestampHns = 0; + const bool gotFrame = session.tryGetNextFrame(&wgcTexture, &wgcTimestampHns); + if (gotFrame) { + if (!latestFrameTexture) { + D3D11_TEXTURE2D_DESC desc{}; + wgcTexture->GetDesc(&desc); + desc.BindFlags = 0; + desc.CPUAccessFlags = 0; + desc.MiscFlags = 0; + if (FAILED(session.device()->CreateTexture2D(&desc, nullptr, &latestFrameTexture))) { + encodeFailed = true; + control.requestStop(); + break; + } + } + // The wedge risk this class exists to avoid: this call + // can block inside the display driver and never return + // (#252, still true of PR #305's DXGI path on some + // hardware). It now does so only on this thread, which + // already owns deciding when to give up -- there is no + // separate WGC callback thread left for it to take a + // lock down with it. + session.context()->CopyResource(latestFrameTexture.Get(), wgcTexture); + latestFrameTimestampHns = wgcTimestampHns; + firstFrameWritten = true; + } else if (!latestFrameTexture) { + // No frame captured yet at all: nothing to encode + // this iteration, and nothing gated on it either (the + // first-frame wait below polls firstFrameWritten + // directly, not a condition variable this thread + // would need to notify). + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } } if (webcamActive) { WebcamFrameSnapshot candidateWebcamFrame; @@ -1024,10 +1176,9 @@ int main(int argc, char* argv[]) { if (lastWebcamTimestampHns >= 0 && webcamTimestampHns <= lastWebcamTimestampHns) { webcamTimestampHns = lastWebcamTimestampHns + nominalWebcamIntervalHns; } - // Capture the sample under `mutex` (the frame copy), but - // submit it to the sink writer OUTSIDE the mutex below - // (issue #115) so a slow WriteSample can't starve the main - // thread's stop-wait. + // Capture the sample here, but submit it to the sink + // writer OUTSIDE this block below (issue #115) so a + // slow WriteSample can't hold up the next frame pull. hasWebcamSample = webcamEncoder.captureBgraSample(webcamFrame, webcamTimestampHns, webcamSample); if (!hasWebcamSample) { encodeFailed = true; @@ -1047,13 +1198,22 @@ int main(int argc, char* argv[]) { std::this_thread::sleep_for(std::chrono::milliseconds(testStallReadbackMs)); } if (latestFrameTexture) { - // Both entry points do their GPU work on latestFrameTexture, - // which must stay serialized (via `mutex`) against the WGC - // frame-arrival callback above, which writes new data into - // the same texture on another thread. Which one is live is - // the encoder's answer, not this struct's request: it falls - // back to the CPU path on its own when the GPU path does - // not fit the machine. + // captureVideoSample/captureDxgiSample perform the GPU + // readback from latestFrameTexture. On the pull-based + // (default) path no lock is needed around it: this thread + // is the only writer of latestFrameTexture too (the + // CopyResource above), so there is no concurrent access + // to serialize against. On the legacy path the WGC + // callback thread also writes latestFrameTexture, under + // frameMutex -- legacyLock is still held here (see its + // declaration above) and is what keeps this readback safe + // in that case. Do not remove the legacy locking on the + // strength of this comment; it describes the default path + // only. + // + // Which entry point is live is the encoder's answer, not + // this struct's request: it falls back to the CPU path on + // its own when the GPU path does not fit the machine. bool captured = false; if (usesDxgiInput) { captured = encoder.captureDxgiSample( @@ -1083,18 +1243,27 @@ int main(int argc, char* argv[]) { } } } + // Explicitly released here, not left to the end of the loop + // iteration: on the legacy path, legacyLock still owns frameMutex + // at this point (unique_lock's scope is its own lifetime, not the + // braces above), and the submission calls below are synchronous + // H.264 encodes that must not run while the WGC callback thread + // is blocked waiting for this same mutex (issue #115). + if (legacyLock.owns_lock()) { + legacyLock.unlock(); + } - // Submit the captured samples to their sink writers OUTSIDE - // `mutex`. IMFSinkWriter::WriteSample runs the H.264 encode - // synchronously and can be slow (especially the software encoder - // fallback used when preferSoftwareEncoder is set), and every - // millisecond it holds `mutex` is a millisecond the WGC frame - // callback spends queued behind it dropping frames (issue #115). + // Submit the captured samples to their sink writers after the + // pull-and-copy block above has finished. IMFSinkWriter:: + // WriteSample runs the H.264 encode synchronously and can be slow + // (especially the software encoder fallback used when + // preferSoftwareEncoder is set); doing it here rather than inside + // the block keeps a slow encode from delaying the next frame pull + // (issue #115). // - // This no longer has anything to do with noticing a stop -- that - // moved off `mutex` entirely (see CaptureControl::stopMutex) after - // issue #252 showed the readback below can wedge inside the lock - // regardless of how briefly WriteSample is held. + // Stop detection has nothing to do with this ordering -- that is + // CaptureControl::stopMutex/stopCv, checked by the loop condition + // above, unrelated to sample submission (issue #252). if (hasWebcamSample && !webcamEncoder.submitVideoSample(webcamSample.Get())) { encodeFailed = true; control.requestStop(); @@ -1254,24 +1423,34 @@ int main(int argc, char* argv[]) { } }); - // The lock covers the wait and the decision, and nothing else. Every - // teardown call below runs outside it, because session.stop() waits for any - // in-flight WGC callback to finish -- and those callbacks block on this very - // mutex. Tearing down while holding it deadlocks the two against each other, - // on the one path the shutdown watchdog does not cover. + // writeVideoFrames is the only caller of session.tryGetNextFrame() now + // (see wgc_session.h), so it has to be running before anything can wait + // for a first frame to arrive -- there is no separate WGC callback thread + // left to deliver one on its own. + if (audioMixer) { + audioMixer->beginTimeline(); + } + control.recordingStartedAt = std::chrono::steady_clock::now(); + startVideoWriter(); + + // firstFrameWritten is set by writeVideoFrames on its own thread; this + // just polls it with the same 10s ceiling the old condition-variable wait + // used. bool firstFrameArrived = false; { - std::unique_lock lock(mutex); - const bool started = control.cv.wait_for(lock, std::chrono::seconds(10), [&] { - return firstFrameWritten.load() || control.stopRequested.load(); - }); - firstFrameArrived = started && firstFrameWritten.load(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!firstFrameWritten.load() && !control.stopRequested.load() && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + firstFrameArrived = firstFrameWritten.load(); } if (!firstFrameArrived) { control.requestStop(); if (stdinThread.joinable()) { stdinThread.detach(); } + stopVideoWriter(); microphoneCapture.stop(); loopbackCapture.stop(); webcamCapture.stop(); @@ -1283,12 +1462,6 @@ int main(int argc, char* argv[]) { return 1; } - if (audioMixer) { - audioMixer->beginTimeline(); - } - control.recordingStartedAt = std::chrono::steady_clock::now(); - startVideoWriter(); - std::cout << "{\"event\":\"recording-started\",\"schemaVersion\":2}" << std::endl; std::cout << "Recording started" << std::endl; @@ -1391,13 +1564,24 @@ int main(int argc, char* argv[]) { // Quiesce the frame producer first. Until WGC is closed, callbacks keep // arriving and keep taking the frame lock, racing the writer's last pass on // the shared D3D context at exactly the moment we can least afford a stall. + // + // Only the legacy push path has a producer thread to quiesce: on the + // default pull path writeVideoFrames is the sole caller of + // tryGetNextFrame(), so its own exit from the loop is the producer + // stopping, and quiesceLegacyCallback() returns immediately with nothing + // to drain. The step still runs and still reports on both paths, so the + // traces line up step for step and `mode` says which one produced them -- + // every report on #252/#460 so far has been read by comparing these lines + // against each other. beginStopStep("wgc-quiesce", stepBudgetMs); // The drain outcome decides the shape of the whole rest of the shutdown: // a callback that never came back makes wgc-session-close skip the device // release, so a report that does not say which happened cannot be read. - const bool wgcDrained = session.quiesceCapture(); + const bool wgcDrained = session.quiesceLegacyCallback(); std::cerr << "[stop-timing] step=wgc-quiesce elapsed_ms=" << stopElapsedMs() - << " drained=" << (wgcDrained ? "true" : "false") << std::endl; + << " drained=" << (wgcDrained ? "true" : "false") + << " mode=" << (legacyFrameCallback ? "legacy-callback" : "pull") + << std::endl; beginStopStep("microphone", stepBudgetMs); microphoneCapture.stop(); logStopStep("microphone"); @@ -1413,17 +1597,68 @@ int main(int argc, char* argv[]) { } logStopStep("audio-mixer"); beginStopStep("video-writer-join", stepBudgetMs); - stopVideoWriter(); - logStopStep("video-writer-join"); + if (wgcDrained) { + stopVideoWriter(); + logStopStep("video-writer-join"); + } else { + // wgc-quiesce already reported the frame callback stuck inside the + // driver (getopenscreen/openscreen#460 on Intel HD 520: a + // CopyResource that never returns), still holding the same + // frame-state `mutex` writeVideoFrames takes for its own + // per-iteration wait -- the one it also needs to notice + // stopRequested. Joining is not a step that can time out here, it is + // one that cannot ever succeed, and this is not the only step that + // assumed it would: encoder.finalize() below resets the very D3D + // device/context a still-blocked writer thread might resume touching + // the moment that lock frees, and quiesceLegacyCallback()/stop() already + // treat "leave everything alone and let process exit reclaim it" as + // the only safe response to exactly this state. So this ends the + // process here, on this thread, rather than pretending the rest of a + // clean shutdown is reachable -- which cost nothing extra before + // today: the same TerminateProcess happened anyway, just + // stepBudgetMs later, once this step's own watchdog gave up waiting + // on a join that could never return. detach() first, not because + // TerminateProcess needs it (it does not touch the C++ runtime, no + // std::thread destructor runs), but so nothing between here and the + // kill can trip over a still-joinable thread. + // + // The fragmented sink writes moof+mdat incrementally, roughly once a + // second, so this is not a new source of loss: whatever was already + // on disk before the callback wedged is on disk regardless of + // whether Finalize() ever runs, on this path or the slower one it + // replaces. + videoWriterThread.detach(); + std::cerr << "[stop-timing] step=video-writer-join elapsed_ms=" << stopElapsedMs() + << " phase=abandoned encode_stage=" << encoder.encodeStage() + << " audio_stage=" << encoder.audioStage() << " reason=frame-callback-stuck" + << std::endl; + std::cout << "{\"event\":\"stop-timeout\",\"schemaVersion\":2,\"step\":\"video-writer-join\"}" + << std::endl; + std::cout.flush(); + std::cerr.flush(); + TerminateProcess(GetCurrentProcess(), 3); + } if (usesDxgiInput) { std::cerr << "[frame-drops] gpu_bridge_contended=" << contendedFrames.load() << std::endl; } - // No frame lock here, and the ordering above is what makes that safe rather - // than incidental: stopVideoWriter() joined the only thread that calls into + // Finalizing before closing the WGC session, not after: MFEncoder holds + // its own ComPtr/ComPtr (see + // mf_encoder.h), separate from WgcSession's, so session.stop() resetting + // WgcSession's pointers would not by itself invalidate what finalize() + // uses -- COM reference counting keeps the underlying device alive until + // MFEncoder releases its own. This ordering does not rely on that: it + // removes the dependency instead of documenting it, so a future change to + // MFEncoder (taking a raw, non-owning pointer, say) cannot silently + // reintroduce a use-after-free. + // + // No frame lock here either, and the ordering above is what makes that + // safe rather than incidental: stopVideoWriter() joined the only thread that calls into // the encoder's GPU readback, and audioMixer->stop() joined the only other // thread that writes to it. MFEncoder's own writerMutex_ deliberately does // NOT cover copyFrameToBuffer, so finalizing before those joins would race - // the staging texture -- do not reorder these. + // the staging texture -- do not reorder these. Reaching this line at all + // means wgcDrained was true above: the branch that was not is a + // TerminateProcess call, not a fallthrough. beginStopStep("encoder-finalize", shutdownBudgetMs); const bool screenFinalized = encoder.finalize(); logStopStep("encoder-finalize"); @@ -1467,8 +1702,13 @@ int main(int argc, char* argv[]) { } } - // Releasing the device goes last: by now no thread can still be holding the - // D3D context. + // Releasing the device goes last, after every encoder that might still + // hold a reference to WgcSession's device has released it via finalize() + // above. By now no thread can still be holding the D3D context: on the + // default pull path writeVideoFrames -- already joined by + // stopVideoWriter() -- was the only caller of tryGetNextFrame()/ + // CopyResource, so its own exit from the while loop is the producer + // stopping; on the legacy path it is wgc-quiesce's drain. beginStopStep("wgc-session-close", stepBudgetMs); session.stop(); logStopStep("wgc-session-close"); diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index 4058ca249..b4dabcc54 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -140,7 +140,7 @@ enum class SinkWriterCreateStage { SoftwareEncoderRegistration, CreateAttributes, DisableHardwareTransforms, - ConfigureDxgiManager, + EnableHardwareTransforms, CreateFile, CreateFragmentedMediaSink, CreateSinkWriter, @@ -248,10 +248,30 @@ HRESULT createSinkWriter( failedStage = SinkWriterCreateStage::DisableHardwareTransforms; return hr; } - } else if (dxgiDeviceManager != nullptr) { - HRESULT hr = MFCreateAttributes(&attributes, 3); + } else { + // Ask for hardware transforms whenever software is not forced -- + // whether or not a DXGI device manager came with the request. + // MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS defaults to FALSE, and + // leaving it unset (the old behaviour on the plain CPU-readback path) + // meant the sink writer never considered a hardware H.264 MFT even + // when one was registered and working: every "default" recording + // landed on the same software encoder forceSoftwareEncoder asks for + // explicitly, on any machine that had not separately opted into + // OPENSCREEN_WGC_ENABLE_DXGI_INPUT (getopenscreen/openscreen#460, + // confirmed by videoEncoderRuntime on real hardware: "default" read + // back "software" until the DXGI path was turned on, on a machine + // whose encoder is hardware-capable either way). + // + // A hardware MFT does not require the D3D manager to accept samples: + // without one it manages its own device and takes system-memory + // samples the same way the software encoder does, which is exactly + // the CPU-readback path this branch also serves. So the attribute is + // set unconditionally here; only the manager itself stays behind the + // null check, since supplying a manager the caller does not have would + // be undefined rather than merely declined. + HRESULT hr = MFCreateAttributes(&attributes, dxgiDeviceManager != nullptr ? 3 : 1); if (FAILED(hr)) { - std::cerr << "ERROR: MFCreateAttributes(DXGI sink writer) failed (hr=0x" + std::cerr << "ERROR: MFCreateAttributes(sink writer) failed (hr=0x" << std::hex << hr << std::dec << ")" << std::endl; failedStage = SinkWriterCreateStage::CreateAttributes; return hr; @@ -260,15 +280,17 @@ HRESULT createSinkWriter( if (FAILED(hr)) { std::cerr << "ERROR: Set MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS(TRUE) failed (hr=0x" << std::hex << hr << std::dec << ")" << std::endl; - failedStage = SinkWriterCreateStage::ConfigureDxgiManager; + failedStage = SinkWriterCreateStage::EnableHardwareTransforms; return hr; } - hr = attributes->SetUnknown(MF_SINK_WRITER_D3D_MANAGER, dxgiDeviceManager); - if (FAILED(hr)) { - std::cerr << "ERROR: Set MF_SINK_WRITER_D3D_MANAGER failed (hr=0x" - << std::hex << hr << std::dec << ")" << std::endl; - failedStage = SinkWriterCreateStage::ConfigureDxgiManager; - return hr; + if (dxgiDeviceManager != nullptr) { + hr = attributes->SetUnknown(MF_SINK_WRITER_D3D_MANAGER, dxgiDeviceManager); + if (FAILED(hr)) { + std::cerr << "ERROR: Set MF_SINK_WRITER_D3D_MANAGER failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + failedStage = SinkWriterCreateStage::EnableHardwareTransforms; + return hr; + } } } @@ -382,6 +404,69 @@ bool resolveStreamSinkIndex(IMFMediaSink* mediaSink, const GUID& majorType, DWOR return false; } +// Did the video stream's encoder MFT actually land on hardware? +// +// BeginWriting() succeeding says nothing about this: even on the "default" +// path (see kVideoEncoderRuntime* in mf_encoder.h), which does now ask for +// MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, that is a request and not a +// guarantee -- Media Foundation is still free to hand the sink writer a +// software MFT when no hardware one is registered or the driver refuses it. +// The only way to know which one it actually picked is to ask the pipeline it +// built, after the fact -- IMFSinkWriterEx::GetTransformForStream walks the +// MFTs the sink writer inserted for a stream, and a hardware MFT instance is +// required to expose MFT_ENUM_HARDWARE_URL_Attribute on its own attribute +// store (not just on the IMFActivate MFTEnumEx returns), which is what +// distinguishes it from a software one at this point. +// +// Every failure path here returns "unknown" rather than guessing: this runs +// after the sink writer is already committed to, so it must never be able to +// fail configureSinkWriterAttempt, and a wrong hardware/software guess in a +// bug report would be worse than an admitted "could not tell." +const char* detectVideoEncoderRuntime(IMFSinkWriter* sinkWriter, DWORD videoStreamIndex) { + Microsoft::WRL::ComPtr sinkWriterEx; + if (FAILED(sinkWriter->QueryInterface(IID_PPV_ARGS(&sinkWriterEx)))) { + return kVideoEncoderRuntimeUnknown; + } + + for (DWORD mftIndex = 0;; mftIndex += 1) { + GUID category{}; + Microsoft::WRL::ComPtr transform; + const HRESULT hr = + sinkWriterEx->GetTransformForStream(videoStreamIndex, mftIndex, &category, &transform); + if (hr == MF_E_INVALIDINDEX) { + // Walked the whole pipeline (converters, the encoder, anything + // else the topology loader inserted) without finding an encoder + // node. Should not happen -- an H.264 stream has to have one -- + // but this is diagnostics code, not the recording path, so an + // unexpected shape is "unknown", not a crash. + return kVideoEncoderRuntimeUnknown; + } + if (FAILED(hr)) { + return kVideoEncoderRuntimeUnknown; + } + if (category != MFT_CATEGORY_VIDEO_ENCODER) { + // A colour converter or similar the sink writer inserted ahead of + // the encoder. Keep walking; the encoder is further down. + continue; + } + + Microsoft::WRL::ComPtr transformAttributes; + if (FAILED(transform->GetAttributes(&transformAttributes))) { + return kVideoEncoderRuntimeUnknown; + } + UINT32 hardwareUrlLength = 0; + const HRESULT hardwareUrlHr = + transformAttributes->GetStringLength(MFT_ENUM_HARDWARE_URL_Attribute, &hardwareUrlLength); + if (SUCCEEDED(hardwareUrlHr)) { + return kVideoEncoderRuntimeHardware; + } + if (hardwareUrlHr == MF_E_ATTRIBUTENOTFOUND) { + return kVideoEncoderRuntimeSoftware; + } + return kVideoEncoderRuntimeUnknown; + } +} + void logSinkWriterCreateFailure( HRESULT sinkWriterHr, const char* createCall, @@ -434,13 +519,15 @@ void setAudioFormat(IMFMediaType* type, UINT32 channels, UINT32 sampleRate, UINT // anything is built from it, not after. bool buildAacOutputType( const AudioInputFormat& audioFormat, + bool skipAacRateSnap, Microsoft::WRL::ComPtr& outputType) { if (audioFormat.sampleRate == 0 || audioFormat.channels == 0 || audioFormat.blockAlign == 0) { std::cerr << "ERROR: Invalid audio input format" << std::endl; return false; } - const AudioInputFormat encoderFormat = makeAacCompatibleAudioFormat(audioFormat); + const AudioInputFormat encoderFormat = + skipAacRateSnap ? audioFormat : makeAacCompatibleAudioFormat(audioFormat); const UINT32 aacBytesPerSecond = 24'000; if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(audio output)")) { @@ -513,6 +600,10 @@ const char* MFEncoder::videoEncoderSelection() const { return videoEncoderSelection_; } +const char* MFEncoder::videoEncoderRuntime() const { + return videoEncoderRuntime_; +} + const char* MFEncoder::containerFormat() const { return containerFormat_; } @@ -600,6 +691,7 @@ bool MFEncoder::initialize( // encoder, never reaching the software encoder the knob is aimed at. useDxgiInput_ = options.useDxgiInput && !options.injectDefaultSinkWriterFailureOnce; videoEncoderSelection_ = kVideoEncoderSelectionDefault; + videoEncoderRuntime_ = kVideoEncoderRuntimeUnknown; if (!succeeded(MFStartup(MF_VERSION), "MFStartup")) { return false; @@ -689,6 +781,7 @@ bool MFEncoder::initialize( audioStreamIndex_ = 0; hasAudioStream_ = false; videoEncoderSelection_ = kVideoEncoderSelectionDefault; + videoEncoderRuntime_ = kVideoEncoderRuntimeUnknown; containerFormat_ = kContainerFormatMp4; }; @@ -704,7 +797,7 @@ bool MFEncoder::initialize( // a construction argument. Null when the recording has no audio, which // is both the common case and a documented one for that call. Microsoft::WRL::ComPtr audioOutputType; - if (audioFormat && !buildAacOutputType(*audioFormat, audioOutputType)) { + if (audioFormat && !buildAacOutputType(*audioFormat, options.skipAacRateSnap, audioOutputType)) { return false; } @@ -768,7 +861,7 @@ bool MFEncoder::initialize( } } - if (audioFormat && !configureAudioStream(*audioFormat)) { + if (audioFormat && !configureAudioStream(*audioFormat, options)) { return false; } @@ -780,7 +873,7 @@ bool MFEncoder::initialize( "SetInputMediaType")) { return false; } - if (useDxgiInput_) { + if (!forceSoftwareEncoder) { applyHardwareRateControl(std::max(1, bitrate)); } if (!succeeded(sinkWriter_->BeginWriting(), "BeginWriting")) { @@ -788,6 +881,7 @@ bool MFEncoder::initialize( } videoEncoderSelection_ = selection; + videoEncoderRuntime_ = detectVideoEncoderRuntime(sinkWriter_.Get(), videoStreamIndex_); containerFormat_ = fragmented ? kContainerFormatFragmentedMp4 : kContainerFormatMp4; return true; }; @@ -854,12 +948,75 @@ bool MFEncoder::initialize( // that used to sit between the two -- now happens before the sink writer // exists. What is left is the input type, which is the same on both containers // and is set on a stream index the caller has already resolved. -bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat) { +HRESULT probeAacPcmRate(UINT32 sampleRate) { + wchar_t dir[MAX_PATH]{}; + GetTempPathW(MAX_PATH, dir); + const std::wstring path = std::wstring(dir) + L"openscreen-wgc-aac-rate-probe-" + + std::to_wstring(GetCurrentProcessId()) + L".mp4"; + DeleteFileW(path.c_str()); + + Microsoft::WRL::ComPtr writer; + HRESULT hr = MFCreateSinkWriterFromURL(path.c_str(), nullptr, nullptr, &writer); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr outputType; + hr = MFCreateMediaType(&outputType); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + outputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + setAudioFormat(outputType.Get(), 2, sampleRate, 16); + outputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, 24'000); + outputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + DWORD streamIndex = 0; + hr = writer->AddStream(outputType.Get(), &streamIndex); + if (FAILED(hr)) { + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr inputType; + hr = MFCreateMediaType(&inputType); + if (FAILED(hr)) { + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + inputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); + setAudioFormat(inputType.Get(), 2, sampleRate, 16); + inputType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, 4); + inputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, sampleRate * 4); + inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + hr = writer->SetInputMediaType(streamIndex, inputType.Get(), nullptr); + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; +} + +bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat, const MFEncoderOptions& options) { if (!sinkWriter_) { return false; } - const AudioInputFormat encoderFormat = makeAacCompatibleAudioFormat(audioFormat); + const AudioInputFormat encoderFormat = + options.skipAacRateSnap ? audioFormat : makeAacCompatibleAudioFormat(audioFormat); + + if (options.injectAacRateProbe) { + const HRESULT probeHr = probeAacPcmRate(encoderFormat.sampleRate); + std::cerr << "TEST-ONLY: AAC rate probe sampleRate=" << encoderFormat.sampleRate + << " hr=0x" << std::hex << probeHr << std::dec << std::endl; + if (!succeeded(probeHr, "SetInputMediaType(audio)")) { + return false; + } + } Microsoft::WRL::ComPtr inputType; if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(audio input)")) { @@ -1204,12 +1361,17 @@ bool MFEncoder::initializeVideoProcessor() { } void MFEncoder::applyHardwareRateControl(int bitrate) { - // The D3D manager switches the sink writer onto a hardware MFT, and those - // default to constant bitrate: a static desktop then spends the full - // configured budget doing nothing, 16.9 Mbps measured against the 1.95 the - // software encoder the CPU path lands on produced for the same screen. Same - // budget, opposite reading of it. Ask for VBR so the GPU path spends what - // the picture costs, which is what users have been getting all along. + // Enabling MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS can hand the sink + // writer a hardware MFT, and those default to constant bitrate: a static + // desktop then spends the full configured budget doing nothing, 16.9 Mbps + // measured against the 1.95 the software encoder produced for the same + // screen. Same budget, opposite reading of it. Ask for VBR so a hardware + // encoder spends what the picture costs, which is what the software + // encoder was already doing. Called whenever hardware transforms were + // requested, DXGI device manager or not (getopenscreen/openscreen#460) -- + // whether the sink writer actually landed on hardware is not knowable + // until after BeginWriting() (see MFEncoder::videoEncoderRuntime()), and + // this call is a no-op on a software MFT that ignores or lacks the knob. // // Best effort on purpose. An encoder that exposes neither knob still // produces a valid recording, and a bitrate we could not pin down is not diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index f8370874c..8d1d6ae6e 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -30,6 +30,17 @@ struct AudioInputFormat { struct MFEncoderOptions { bool preferSoftwareEncoder = false; bool injectDefaultSinkWriterFailureOnce = false; + // Test-only. Keep the sample rate passed into initialize() instead of + // running makeAacCompatibleAudioFormat again. OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP + // used to change only the JSON log: buildAacOutputType / configureAudioStream + // snapped 96 kHz back to 48 kHz before SetInputMediaType, so the fail path + // was unreachable. + bool skipAacRateSnap = false; + // Test-only. Before the production sink writer sees the audio type, run + // MFCreateSinkWriterFromURL + SetInputMediaType on the encoder rate. Illegal + // AAC rates fail that probe with MF_E_INVALIDMEDIATYPE (0xc00d36b4). Does + // not invent the HRESULT — it calls the API. + bool injectAacRateProbe = false; // A request, never a requirement. Every step of the GPU path degrades to // the CPU readback rather than failing the recording, so a machine without // a hardware H.264 encoder, without NV12 video-processor output, or with a @@ -42,6 +53,28 @@ constexpr const char* kVideoEncoderSelectionDefault = "default"; constexpr const char* kVideoEncoderSelectionSoftwarePreferred = "software-preferred"; constexpr const char* kVideoEncoderSelectionSoftwareFallback = "software-fallback"; +// Whether BeginWriting() actually landed on a hardware-accelerated H.264 MFT. +// +// videoEncoderSelection() above says which *path* initialize() took -- +// whether the DXGI GPU pipeline was asked for, or software was forced -- but +// none of those labels says what Media Foundation itself picked, and that +// matters even now that createSinkWriter asks for hardware transforms on +// every path but the forced-software one: MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS +// asks, it does not guarantee -- a machine with no hardware H.264 MFT +// registered, or one whose driver refuses it, still lands on software. That +// gap is exactly what this exists to close for a bug report: "default" alone +// cannot tell a real hardware encode apart from software Media Foundation +// picked anyway, which was the whole ambiguity behind a slow-CPU stop timeout +// (getopenscreen/openscreen#460) before this field existed. +constexpr const char* kVideoEncoderRuntimeHardware = "hardware"; +constexpr const char* kVideoEncoderRuntimeSoftware = "software"; +// Introspection itself failed (no IMFSinkWriterEx, no encoder node found in +// the resolved topology, GetAttributes refused). Reported as its own value +// rather than guessed into hardware or software, because a bug report that +// cannot tell "we checked and it's software" from "we couldn't check" would +// draw the wrong conclusion either way. +constexpr const char* kVideoEncoderRuntimeUnknown = "unknown"; + // Which MP4 flavour the recording was actually written in. The fragmented sink // writes a self-describing moof+mdat pair roughly every second, so a helper the // shutdown watchdog force-exits leaves a file that plays up to the last @@ -97,6 +130,9 @@ class MFEncoder { bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns); bool finalize(); const char* videoEncoderSelection() const; + // Best-effort, read only after initialize() returns true. See the + // kVideoEncoderRuntime* constants above for what each value means. + const char* videoEncoderRuntime() const; // Which container initialize() settled on, which is not necessarily the one // it asked for: the fragmented sink degrades to the plain one rather than // failing a recording. A bug report that cannot tell the two apart cannot @@ -150,7 +186,7 @@ class MFEncoder { // built before the sink writer exists (buildAacOutputType in the .cpp), // because MFCreateFMPEG4MediaSink takes both output types at construction: // a fragmented sink has all its streams before anything can be added to it. - bool configureAudioStream(const AudioInputFormat& audioFormat); + bool configureAudioStream(const AudioInputFormat& audioFormat, const MFEncoderOptions& options); void releaseSinkWriter(); Microsoft::WRL::ComPtr sinkWriter_; @@ -202,5 +238,6 @@ class MFEncoder { bool finalized_ = false; bool useDxgiInput_ = false; const char* videoEncoderSelection_ = kVideoEncoderSelectionDefault; + const char* videoEncoderRuntime_ = kVideoEncoderRuntimeUnknown; const char* containerFormat_ = kContainerFormatMp4; }; diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp index 76649a990..b75d0819e 100644 --- a/electron/native/wgc-capture/src/wgc_session.cpp +++ b/electron/native/wgc-capture/src/wgc_session.cpp @@ -237,7 +237,6 @@ bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) { return false; } - frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived}); return true; } @@ -261,15 +260,9 @@ bool WgcSession::initialize(HWND window, int fps, bool captureCursor) { return false; } - frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived}); return true; } -void WgcSession::setFrameCallback(FrameCallback callback) { - std::scoped_lock lock(callbackMutex_); - frameCallback_ = std::move(callback); -} - bool WgcSession::start() { if (!session_) { return false; @@ -282,12 +275,128 @@ bool WgcSession::start() { return true; } -bool WgcSession::quiesceCapture(int drainTimeoutMs) { +bool WgcSession::tryGetNextFrame(ID3D11Texture2D** outTexture, int64_t* outTimestampHns) { + if (!framePool_) { + return false; + } + + // TryGetNextFrame() and frame.Close() are the only WGC calls this makes; + // neither performs the GPU copy itself, so neither is where a wedge in + // #252 was ever observed. The copy (CopyResource, on whatever the caller + // does with *outTexture) is the caller's own doing on the caller's own + // thread -- this class has no thread of its own left to hang on their + // behalf. + auto frame = framePool_.TryGetNextFrame(); + if (!frame) { + return false; + } + + auto surface = frame.Surface(); + auto access = surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>(); + Microsoft::WRL::ComPtr texture; + HRESULT hr = access->GetInterface(__uuidof(ID3D11Texture2D), reinterpret_cast(texture.GetAddressOf())); + if (FAILED(hr) || !texture) { + return false; + } + + // Closing the previous frame here (rather than right after this class + // copied out of it) returns it to the pool only once the caller has had a + // full interval to read the one before that -- the pool has 2 buffers, so + // closing eagerly would let WGC recycle a buffer the caller might still + // be mid-CopyResource on across the two-call boundary. currentFrame_ + // holds the reference that keeps *outTexture valid until this class's + // next call or stop() closes it. + currentFrame_ = frame; + + *outTexture = texture.Get(); + *outTimestampHns = timeSpanToHns(frame.SystemRelativeTime()); + return true; +} + +void WgcSession::setFrameCallback(FrameCallback callback) { + if (!legacyCallbackRegistered_ && framePool_) { + frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived}); + legacyCallbackRegistered_ = true; + } + std::scoped_lock lock(callbackMutex_); + frameCallback_ = std::move(callback); +} + +void WgcSession::onFrameArrived( + wgcap::Direct3D11CaptureFramePool const& sender, + wf::IInspectable const&) { + // Scoped rather than a bare decrement at the end, for two reasons: a + // callback that left by exception would otherwise strand + // quiesceLegacyCallback()'s drain forever, and the guard has to outlive + // every pool-owned object this handler touches -- dropping the count + // first would let quiesce return and close the frame pool while this + // handler still holds a reference into it. + struct InFlightGuard { + std::atomic& counter; + ~InFlightGuard() { + counter -= 1; + } + }; + + // Captured and counted before TryGetNextFrame(), not after: this handler + // starts touching the pool (TryGetNextFrame, Surface(), GetInterface()) + // immediately below, and none of that is safe to run concurrently with + // framePool_.Close(). Counting only after those calls succeeded left a + // window where quiesceLegacyCallback() could see callbacksInFlight_ == 0 + // and return while this handler was still mid-frame -- registering the + // guard first, before anything pool-related, closes that window instead + // of narrowing it. + // + // Returns here, before incrementing the counter or touching the pool, if + // frameCallback_ is already null: there is nothing to do with a frame in + // that case, so the handler should not acquire one. This also means a + // handler that starts after quiesceLegacyCallback() has cleared + // frameCallback_ is never counted at all -- which is fine, since it never + // reaches the pool either. + FrameCallback callback; + { + std::scoped_lock lock(callbackMutex_); + callback = frameCallback_; + if (!callback) { + return; + } + // Counted under the same lock quiesceLegacyCallback() clears the + // callback under, so once it has cleared it no new handler can start + // and the counter it then drains cannot go back up. + callbacksInFlight_ += 1; + } + InFlightGuard guard{callbacksInFlight_}; + + auto frame = sender.TryGetNextFrame(); + if (!frame) { + return; + } + + auto surface = frame.Surface(); + auto access = surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>(); + Microsoft::WRL::ComPtr texture; + HRESULT hr = access->GetInterface(__uuidof(ID3D11Texture2D), reinterpret_cast(texture.GetAddressOf())); + if (FAILED(hr) || !texture) { + frame.Close(); + return; + } + + // callback is never null here: the only path that reaches this point + // returned earlier if frameCallback_ was null when captured. + callback(texture.Get(), timeSpanToHns(frame.SystemRelativeTime())); + frame.Close(); +} + +bool WgcSession::quiesceLegacyCallback(int drainTimeoutMs) { if (quiesced_) { return callbacksInFlight_.load() == 0; } quiesced_ = true; + if (!legacyCallbackRegistered_) { + return true; + } + try { if (framePool_) { framePool_.FrameArrived(frameArrivedToken_); @@ -297,20 +406,21 @@ bool WgcSession::quiesceCapture(int drainTimeoutMs) { // to abandon the rest of the shutdown. } { - // Drop the callback under the same lock onFrameArrived copies it under, - // so any handler that has not read it yet becomes a no-op... + // Drop the callback under the same lock onFrameArrived copies it + // under, so any handler that has not read it yet becomes a no-op... std::scoped_lock lock(callbackMutex_); frameCallback_ = nullptr; } - // ...then wait out the handlers that already read it. Without this, stop() - // could Reset() the D3D context while a callback was still issuing - // CopyResource on it. + // ...then wait out the handlers that already read it. Without this, + // stop() could Reset() the D3D context while a callback was still + // issuing CopyResource on it. // // Bounded, because a callback wedged inside the display driver never - // finishes and this runs on paths that have no watchdog above them (the - // first-frame timeout in main.cpp). Giving up is reported rather than - // papered over: the caller keeps the device alive instead, which leaks it - // until the process exits and is the lesser of the two failures. + // finishes (this is #252 -- the exact failure this legacy path is kept + // around to let a user opt back into, so its own known weakness needs no + // further comment here). Giving up is reported rather than papered over: + // the caller keeps the device alive instead, which leaks it until the + // process exits and is the lesser of the two failures. const auto drainDeadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(drainTimeoutMs); while (callbacksInFlight_.load() > 0) { @@ -321,13 +431,38 @@ bool WgcSession::quiesceCapture(int drainTimeoutMs) { } std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + return true; +} + +void WgcSession::stop() { + if (!started_ && !framePool_) { + return; + } + + if (legacyCallbackRegistered_ && !quiesceLegacyCallback()) { + // A callback is still inside the driver holding this context. + // Releasing it now would pull the device out from under a live + // CopyResource, so leak it and let process exit reclaim it. This is + // the exact hang class the pull-based default avoids; it is only + // reachable via OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK=1. + return; + } // Close() is a C++/WinRT projection and throws hresult_error on failure. // Letting that escape would take the process down through std::terminate - // mid-shutdown, discarding a recording that is already finalized by the time - // this runs. There is nothing to do about a capture session that refuses to - // close except stop caring about it. + // mid-shutdown, discarding a recording that is already finalized by the + // time this runs. There is nothing to do about a capture session that + // refuses to close except stop caring about it. + // + // On the pull-based (default) path, there is no other thread that could + // be mid-copy on currentFrame_'s texture when this runs: the caller only + // ever calls tryGetNextFrame() and stop() from its own thread, so by the + // time stop() is reached whatever the caller was doing with the last + // texture it read is already done. On the legacy path, the + // quiesceLegacyCallback() call above already established the same + // invariant before falling through to here. try { + currentFrame_ = nullptr; if (session_) { session_.Close(); } @@ -343,70 +478,12 @@ bool WgcSession::quiesceCapture(int drainTimeoutMs) { session_ = nullptr; framePool_ = nullptr; started_ = false; - return true; -} - -void WgcSession::stop() { - if (!quiesceCapture()) { - // A callback is still inside the driver holding this context. Releasing - // it now would pull the device out from under a live CopyResource, so - // leak it and let process exit reclaim it. - return; - } item_ = nullptr; winrtDevice_ = nullptr; d3dContext_.Reset(); d3dDevice_.Reset(); } -void WgcSession::onFrameArrived( - wgcap::Direct3D11CaptureFramePool const& sender, - wf::IInspectable const&) { - auto frame = sender.TryGetNextFrame(); - if (!frame) { - return; - } - - auto surface = frame.Surface(); - auto access = surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>(); - Microsoft::WRL::ComPtr texture; - HRESULT hr = access->GetInterface(__uuidof(ID3D11Texture2D), reinterpret_cast(texture.GetAddressOf())); - if (FAILED(hr) || !texture) { - return; - } - - FrameCallback callback; - { - std::scoped_lock lock(callbackMutex_); - callback = frameCallback_; - if (callback) { - // Counted under the same lock quiesceCapture() clears the callback - // under, so once it has cleared it no new callback can start and - // the counter it then drains cannot go back up. - callbacksInFlight_ += 1; - } - } - - if (callback) { - // Scoped rather than a bare decrement after the call, for two reasons: - // a callback that left by exception would otherwise strand - // quiesceCapture()'s drain forever, and the guard has to outlive - // frame.Close() -- dropping the count first would let quiesce return and - // close the frame pool while this handler is still closing a frame that - // pool owns. - struct InFlightGuard { - std::atomic& counter; - ~InFlightGuard() { - counter -= 1; - } - } guard{callbacksInFlight_}; - callback(texture.Get(), timeSpanToHns(frame.SystemRelativeTime())); - frame.Close(); - return; - } - frame.Close(); -} - int WgcSession::captureWidth() const { return width_; } diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h index 33aba29b4..35eb25a54 100644 --- a/electron/native/wgc-capture/src/wgc_session.h +++ b/electron/native/wgc-capture/src/wgc_session.h @@ -10,9 +10,38 @@ #include #include +#include #include #include +// Frame delivery defaults to pull-based, not the WGC FrameArrived event: the +// caller's own thread polls tryGetNextFrame() on its own schedule and does +// the GPU copy itself. The reason is this codebase's threading model, not +// another project's precedent: a FrameArrived handler runs on a WGC-owned +// thread, so any lock a caller takes to synchronize that handler with its +// own pipeline ends up held by a thread the caller does not control. If the +// copy wedges inside the display driver -- which happens on real hardware, +// not hypothetically (see #252 and #460) -- that lock is gone until the +// process exits, and every other thread that ever needs it hangs too, +// however briefly it would otherwise have held it. Pulling on the caller's +// own thread means a wedged copy only ever blocks the one thread already +// responsible for deciding when to give up on it; nothing else can be +// dragged in. +// +// Chromium's WGC capturer also pulls rather than handling FrameArrived, but +// not for this reason -- its comment there is about avoiding a +// DispatcherQueue, and it runs a 1-buffer pool because a dropped frame costs +// a screen-sharing viewer nothing. We record, where a dropped frame is a +// defect in a file someone keeps, so none of its sizing carries over here. +// +// The old FrameArrived-callback path (setFrameCallback/onFrameArrived) is +// kept alongside it, selected by OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK (see +// main.cpp), as a rollback lever: if the pull-based path regresses on some +// hardware/driver combination this was not tested against, a user or +// maintainer can force the previously-shipped behavior back on without +// waiting for a new release. It carries its own known failure mode (#252) +// and is not a recommended default -- remove it once the pull-based path has +// enough field time to retire the flag. class WgcSession { public: using FrameCallback = std::function; @@ -25,16 +54,26 @@ class WgcSession { bool initialize(HMONITOR monitor, int fps, bool captureCursor); bool initialize(HWND window, int fps, bool captureCursor); - void setFrameCallback(FrameCallback callback); bool start(); - // Stops frame delivery and waits out any callback already running, without - // touching the D3D device. Split out of stop() so a caller can quiesce the - // producer early in a shutdown and only release the device once nothing can - // still be using it. Idempotent; stop() calls it. - // - // Returns false if a callback was still running when `drainTimeoutMs` - // expired -- releasing the device after that is unsafe, so stop() skips it. - bool quiesceCapture(int drainTimeoutMs = 5000); + // Returns the most recently arrived frame's texture and timestamp, or + // false if none is available since the last call. The returned pointer + // is only valid until the next tryGetNextFrame() call or stop() -- copy + // out of it (e.g. via CopyResource) before either. Do not mix with + // setFrameCallback() on the same session. + bool tryGetNextFrame(ID3D11Texture2D** outTexture, int64_t* outTimestampHns); + + // Legacy push-based path (OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK=1 only). + // callback runs on a WGC-owned thread inside FrameArrived and may be + // invoked concurrently with stop()/quiesceLegacyCallback() from the + // caller's thread -- see onFrameArrived's locking. Do not mix with + // tryGetNextFrame() on the same session. + void setFrameCallback(FrameCallback callback); + // Stops frame delivery and waits out any callback already running, + // without touching the D3D device. Only meaningful after + // setFrameCallback(); a no-op on the pull-based path. Returns false if a + // callback was still running when drainTimeoutMs expired -- releasing + // the device after that is unsafe, so stop() skips it in that case. + bool quiesceLegacyCallback(int drainTimeoutMs = 5000); void stop(); int captureWidth() const; @@ -57,10 +96,18 @@ class WgcSession { winrt::Windows::Graphics::Capture::GraphicsCaptureItem item_{nullptr}; winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool framePool_{nullptr}; winrt::Windows::Graphics::Capture::GraphicsCaptureSession session_{nullptr}; + // Keeps the most recent frame's WinRT wrapper (and therefore its + // pool-owned texture) alive between tryGetNextFrame() calls: the pool is + // created with 2 buffers, so holding this reference is what keeps the + // texture valid for the caller to read from until the next call reclaims + // it. + winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame currentFrame_{nullptr}; + // Legacy push-based path state; unused unless setFrameCallback() is called. winrt::event_token frameArrivedToken_{}; FrameCallback frameCallback_; std::mutex callbackMutex_; std::atomic callbacksInFlight_ = 0; + bool legacyCallbackRegistered_ = false; bool quiesced_ = false; int width_ = 0; int height_ = 0; diff --git a/electron/preload.ts b/electron/preload.ts index 6aff16407..7873bef90 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -276,6 +276,12 @@ contextBridge.exposeInMainWorld("electronAPI", { openVideoFilePicker: () => { return ipcRenderer.invoke("open-video-file-picker"); }, + openAudioFilePicker: () => { + return ipcRenderer.invoke("open-audio-file-picker"); + }, + saveRecordedVoiceover: (data: ArrayBuffer) => { + return ipcRenderer.invoke("save-recorded-voiceover", data); + }, setCurrentVideoPath: (path: string) => { return ipcRenderer.invoke("set-current-video-path", path); }, diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts new file mode 100644 index 000000000..24e9694c8 --- /dev/null +++ b/electron/stt/extractAudio.test.ts @@ -0,0 +1,141 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const spawnMock = vi.fn(); +const resolveFfmpegMock = vi.fn<() => string | null>(); + +vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) })); +vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() })); + +const { extractMono16kPcm, FfmpegUnavailableError, NoAudioTrackError } = await import( + "./extractAudio" +); +const { STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import("./transcriptionContract"); + +/** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */ +function fakeChild() { + const child = new EventEmitter() as EventEmitter & { + stdout: PassThrough; + stderr: PassThrough; + kill: ReturnType; + }; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = vi.fn(); + return child; +} + +/** The little-endian float32 bytes ffmpeg would emit for `values`. */ +function f32le(values: number[]): Buffer { + const buf = Buffer.alloc(values.length * 4); + values.forEach((v, i) => buf.writeFloatLE(v, i * 4)); + return buf; +} + +beforeEach(() => { + vi.clearAllMocks(); + resolveFfmpegMock.mockReturnValue("/usr/bin/ffmpeg"); +}); + +describe("extractMono16kPcm", () => { + it("asks ffmpeg for exactly what whisper wants", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/a.mp3"); + child.stdout.end(f32le([0.5])); + child.emit("close", 0); + await promise; + + const args = spawnMock.mock.calls[0][1] as string[]; + // Mono, 16 kHz, float32 little-endian, no video. Anything else and whisper is + // reading the samples wrong rather than failing loudly. + expect(args).toContain("-vn"); + expect(args.join(" ")).toContain("-ac 1"); + expect(args.join(" ")).toContain("-ar 16000"); + expect(args.join(" ")).toContain("-f f32le"); + }); + + it("decodes the samples ffmpeg writes", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/a.mp3"); + child.stdout.end(f32le([0, 0.5, -0.25])); + child.emit("close", 0); + + const out = await promise; + expect(Array.from(out)).toEqual([0, 0.5, -0.25]); + }); + + it("carries a float split across two chunks instead of dropping it", async () => { + // THE defect worth a test here: stdout chunk boundaries do not respect sample + // boundaries. Dropping the partial tail would shift every following sample and + // detune the whole track — audible, and invisible in a length check. + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/a.mp3"); + const bytes = f32le([0.25, -0.75, 1]); + child.stdout.write(bytes.subarray(0, 6)); // one whole float + half of the next + child.stdout.write(bytes.subarray(6)); + child.stdout.end(); + child.emit("close", 0); + + const out = await promise; + expect(Array.from(out)).toEqual([0.25, -0.75, 1]); + }); + + it("reports a file with no audio track", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/silent.mp4"); + child.stderr.end("Stream map '0:a' matches no streams"); + child.stdout.end(); + child.emit("close", 1); + + await expect(promise).rejects.toBeInstanceOf(NoAudioTrackError); + }); + + it("keeps the samples when ffmpeg exits non-zero AFTER writing audio", async () => { + // A truncated file still yields usable audio; throwing it away would lose a + // transcript over a trailing byte. + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/truncated.mp3"); + child.stdout.end(f32le([0.1, 0.2])); + child.emit("close", 1); + + expect(Array.from(await promise)).toEqual([expect.closeTo(0.1, 6), expect.closeTo(0.2, 6)]); + }); + + it("refuses with the marker the renderer falls back on when ffmpeg is missing", async () => { + // The string is the contract across the IPC boundary, which drops the class. + resolveFfmpegMock.mockReturnValue(null); + await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toBeInstanceOf(FfmpegUnavailableError); + await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow( + STT_NATIVE_EXTRACTION_UNAVAILABLE, + ); + expect(spawnMock).not.toHaveBeenCalled(); + }); + + it("kills the child when the caller aborts", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const controller = new AbortController(); + const promise = extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal }); + controller.abort(); + + await expect(promise).rejects.toMatchObject({ name: "AbortError" }); + // Not merely stopping to await: ffmpeg would keep decoding a long file for + // minutes, which is the same leak the STT cancel path exists to prevent. + expect(child.kill).toHaveBeenCalled(); + }); + + it("does not spawn at all when the signal is already aborted", async () => { + const controller = new AbortController(); + controller.abort(); + await expect( + extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(spawnMock).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts new file mode 100644 index 000000000..08168d21e --- /dev/null +++ b/electron/stt/extractAudio.ts @@ -0,0 +1,152 @@ +// Native mono-16k extraction for transcription, in the main process. +// +// The renderer used to do this: `extractMono16kFromVideoUrl` read the whole media +// into a `File`, took an `arrayBuffer()`, handed a `slice(0)` copy to +// `decodeAudioData`, and resampled the result to mono 16k — all of it on the UI +// thread, all of it before whisper ever saw a sample. On a four-minute bed that is +// ~86 MB of decoded float32 plus two copies of the encoded bytes, and it froze the +// editor at open. The inference itself was never the problem: it runs in +// `whisper-stt-server`, in its own process, on the GPU. +// +// So this is the same remedy `useAudioPeaks` already got (see +// `electron/media/audioPeaks.ts`, and the note there about WHICH ffmpeg is packaged +// on Windows): let ffmpeg do it, in the main process, streaming. It costs +// `durationSec * 16000 * 4` bytes — 15.7 MB for that same four-minute bed — and the +// renderer never allocates any of it. +// +// It is deliberately NOT cached on disk, unlike peaks. Peaks are re-read on every +// project open; extraction feeds one transcription, whose RESULT is what gets +// persisted (`document.transcripts[]`). Caching the PCM would trade disk for work +// that is already never repeated. + +import { spawn } from "node:child_process"; +import { resolveFfmpeg } from "../media/audioPeaks"; +import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; + +/** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */ +const SAMPLE_RATE = 16_000; + +/** Past this, it is not a recording — it is a wedged ffmpeg. Matches the peaks path. */ +const EXTRACT_TIMEOUT_MS = 60_000; + +/** + * Thrown when no ffmpeg can be resolved, so the caller can fall back to the + * renderer pipeline rather than failing the transcription outright. A distinct type + * because "there is no ffmpeg here" and "this file has no audio" want opposite + * responses: fall back, versus report a permanent failure for this asset. + */ +export class FfmpegUnavailableError extends Error { + constructor() { + super(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary for native audio extraction`); + this.name = "FfmpegUnavailableError"; + } +} + +/** A media with no decodable audio track. Permanent for that file. */ +export class NoAudioTrackError extends Error { + constructor(filePath: string, detail: string) { + super(`No decodable audio in ${filePath}${detail ? `: ${detail}` : ""}`); + this.name = "NoAudioTrackError"; + } +} + +/** + * Decode `filePath` to mono 16 kHz float samples. + * + * Streams `f32le` straight off ffmpeg's stdout, so the only full-size allocation is + * the result itself. Chunk boundaries do not respect sample boundaries — a 4-byte + * float can straddle two `data` events — so a partial tail is carried into the next + * chunk rather than dropped, which would shift every following sample and detune the + * whole track. + */ +export async function extractMono16kPcm( + filePath: string, + options: { signal?: AbortSignal } = {}, +): Promise { + const ffmpeg = resolveFfmpeg(); + if (!ffmpeg) throw new FfmpegUnavailableError(); + if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError"); + + const child = spawn( + ffmpeg, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + filePath, + "-vn", + "-ac", + "1", + "-ar", + String(SAMPLE_RATE), + "-f", + "f32le", + "-", + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + return new Promise((resolve, reject) => { + const chunks: Float32Array[] = []; + let total = 0; + /** Bytes of a float that arrived split across two chunks. */ + let carry: Buffer | null = null; + let stderr = ""; + let settled = false; + + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + fn(); + }; + + const timer = setTimeout(() => { + child.kill("SIGKILL"); + finish(() => + reject(new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`)), + ); + }, EXTRACT_TIMEOUT_MS); + + const onAbort = () => { + child.kill("SIGKILL"); + finish(() => reject(new DOMException("Aborted", "AbortError"))); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + + child.stdout.on("data", (c: Buffer) => { + const buf = carry ? Buffer.concat([carry, c]) : c; + const usable = buf.length - (buf.length % 4); + if (usable > 0) { + // Copy rather than view: a Buffer's memory is rarely 4-byte aligned, and + // `byteOffset` is almost never 0. + const view = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + usable)); + chunks.push(view); + total += view.length; + } + carry = usable < buf.length ? Buffer.from(buf.subarray(usable)) : null; + }); + child.stderr.on("data", (c: Buffer) => { + stderr = (stderr + c.toString()).slice(-2048); + }); + child.once("error", (err) => finish(() => reject(err))); + child.once("close", (code) => { + // A file with no audio track exits non-zero, and so does a corrupt one. The + // caller treats both the same way — this asset will not transcribe — so they + // share an error type; `stderr` carries which it was. + if (code !== 0 && total === 0) { + finish(() => reject(new NoAudioTrackError(filePath, stderr.trim()))); + return; + } + const out = new Float32Array(total); + let at = 0; + for (const part of chunks) { + out.set(part, at); + at += part.length; + } + finish(() => resolve(out)); + }); + }); +} diff --git a/electron/stt/index.ts b/electron/stt/index.ts index c250ea4ac..a20f4e577 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { app, type IpcMain } from "electron"; import { planChunks } from "./chunking"; +import { extractMono16kPcm } from "./extractAudio"; import { ensureModels, modelPaths } from "./modelManager"; import type { SttPhraseSegment, @@ -103,6 +104,13 @@ export class SttManager { */ private cancelEpoch = 0; + /** + * The extraction in flight, if any. `cancelEpoch` alone stops the CHUNK loop, which is + * checked between chunks — so a cancel during the decode left ffmpeg running to + * completion on a file that can be hours long, and the user saw nothing stop. + */ + private extraction: AbortController | null = null; + /** * Attach a sink for the renderer status channel; returns its detach function. * @@ -134,6 +142,7 @@ export class SttManager { */ cancel(): void { this.cancelEpoch++; + this.extraction?.abort(); } /** @@ -237,12 +246,34 @@ export class SttManager { } /** Transcribe a whole recording, chunk by chunk, reporting progress as it goes. */ + /** Decode `sourcePath` into the samples `transcribe` needs. */ + private async extract(req: SttTranscribeRequest): Promise { + if (!req.sourcePath) { + throw new Error("stt:transcribe needs either `samples` or `sourcePath`"); + } + const controller = new AbortController(); + this.extraction = controller; + try { + return await extractMono16kPcm(req.sourcePath, { signal: controller.signal }); + } finally { + // Only if it is still ours: a cancel that started a new run must not have its + // controller cleared by the old one unwinding. + if (this.extraction === controller) this.extraction = null; + } + } + async transcribe(req: SttTranscribeRequest): Promise { await this.init(); const epoch = this.cancelEpoch; - const totalSec = req.samples.length / SAMPLE_RATE; - const chunks = planChunks(req.samples, SAMPLE_RATE); + // Extraction is part of the run, and on a long file it is the part the user used + // to watch the editor freeze through. Doing it here means the renderer hands over + // a path and gets segments back, holding none of the audio. No new status phase: + // the caller already reports "extracting-audio" around this call, and the work + // simply moved to the other side of the IPC. + const samples = req.samples ?? (await this.extract(req)); + const totalSec = samples.length / SAMPLE_RATE; + const chunks = planChunks(samples, SAMPLE_RATE); this.emit({ phase: "transcribe", completedSec: 0, totalSec }); const segments: SttPhraseSegment[] = []; @@ -285,7 +316,7 @@ export class SttManager { if (this.cancelEpoch !== epoch) throw cancelledError(); const offsetSec = chunk.startSample / SAMPLE_RATE; const result = await this.transcribeChunk( - req.samples.subarray(chunk.startSample, chunk.endSample), + samples.subarray(chunk.startSample, chunk.endSample), language, ).catch((error) => { if (error instanceof Error && error.name === "AbortError") throw error; diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index cbce5a141..d2e67ae11 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -106,7 +106,24 @@ export interface SttStatusEvent { /** IPC request: renderer → main. */ export interface SttTranscribeRequest { - samples: Float32Array; + /** + * Mono-16k samples the CALLER decoded. Optional since native extraction landed: + * pass `sourcePath` instead and the main process decodes with ffmpeg, off the UI + * thread and without the renderer ever holding the audio. Kept for the caption + * path, which already has samples in hand and has no file to point at. + * + * Exactly one of `samples` / `sourcePath` is required. + */ + samples?: Float32Array; + /** + * A media file for the main process to decode itself (ffmpeg -> mono 16k f32). + * Preferred: the renderer's own pipeline read the whole file, copied it twice and + * resampled it on the UI thread, which is what froze the editor at open. + * + * The caller falls back to its own decode when this cannot be honoured — see + * `FfmpegUnavailableError`. + */ + sourcePath?: string; /** * ISO 639-1 language code (e.g. "en", "fr"). Omit / `"auto"` to let Whisper detect. * The spec locks language detection on by default; we only honour an explicit value. @@ -114,6 +131,17 @@ export interface SttTranscribeRequest { language?: string; } +/** + * Marker carried in the error message when the main process cannot decode a + * `sourcePath` because no ffmpeg is resolvable on this install. + * + * A string rather than an error class because this crosses `ipcRenderer.invoke`, + * which reconstructs a plain `Error` from the message and drops the prototype and + * the `name`. Exported so neither side spells it out by hand — a fallback keyed on + * a literal typed twice is a fallback that silently stops working. + */ +export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable"; + /** IPC response: main → renderer. */ export interface SttTranscribeResponse { segments: SttPhraseSegment[]; diff --git a/electron/update-settings.test.ts b/electron/update-settings.test.ts new file mode 100644 index 000000000..4050987c6 --- /dev/null +++ b/electron/update-settings.test.ts @@ -0,0 +1,44 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parseUpdateMode } from "./background-update"; +import { loadUpdateMode, saveUpdateMode, updateSettingsPath } from "./update-settings"; + +const temps: string[] = []; + +afterEach(() => { + for (const dir of temps) rmSync(dir, { recursive: true, force: true }); + temps.length = 0; +}); + +function tmp(): string { + const dir = mkdtempSync(path.join(os.tmpdir(), "os-update-settings-")); + temps.push(dir); + return dir; +} + +describe("update settings", () => { + it("round-trips a saved mode through load", () => { + const dir = tmp(); + saveUpdateMode(dir, "download-and-install"); + expect(loadUpdateMode(dir)).toBe("download-and-install"); + }); + + it("defaults to notify when nothing was ever saved", () => { + expect(loadUpdateMode(tmp())).toBe("notify"); + }); + + it("falls back to notify on a corrupt settings file instead of throwing", () => { + const dir = tmp(); + writeFileSync(updateSettingsPath(dir), '{"mode": not-even-json !!}'); + expect(loadUpdateMode(dir)).toBe("notify"); + }); + + it("refuses garbage mode values rather than trusting the file", () => { + for (const garbage of ["install-silently", "", 42, null, { mode: "download" }]) { + expect(parseUpdateMode(garbage)).toBe("notify"); + } + expect(parseUpdateMode("download")).toBe("download"); + }); +}); diff --git a/electron/update-settings.ts b/electron/update-settings.ts new file mode 100644 index 000000000..9e78b2c30 --- /dev/null +++ b/electron/update-settings.ts @@ -0,0 +1,26 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { DEFAULT_UPDATE_MODE, parseUpdateMode, type UpdateMode } from "./background-update"; + +export function updateSettingsPath(userData: string): string { + return path.join(userData, "update-settings.json"); +} + +export function loadUpdateMode(userData: string): UpdateMode { + const file = updateSettingsPath(userData); + if (!existsSync(file)) return DEFAULT_UPDATE_MODE; + try { + const raw = JSON.parse(readFileSync(file, "utf8")) as { mode?: unknown }; + return parseUpdateMode(raw.mode); + } catch { + return DEFAULT_UPDATE_MODE; + } +} + +export function saveUpdateMode(userData: string, mode: UpdateMode): void { + try { + writeFileSync(updateSettingsPath(userData), `${JSON.stringify({ mode })}\n`, "utf8"); + } catch { + // Best-effort; a failed write must not block the tray click. + } +} diff --git a/electron/windows.ts b/electron/windows.ts index 4b5ceb7fe..a9f51e41e 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -1,6 +1,13 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { BrowserWindow, ipcMain, screen } from "electron"; +import { app, BrowserWindow, ipcMain, screen } from "electron"; +import { + clampRectToWorkArea, + loadEditorWindowState, + resolveEditorCreation, + saveEditorWindowState, + shouldTrackEditorWindow, +} from "./editorWindowState"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -392,10 +399,18 @@ export function createHudOverlayWindow(): BrowserWindow { */ export function createEditorWindow(query: Record = {}): BrowserWindow { const isMac = process.platform === "darwin"; + const persist = shouldTrackEditorWindow(query); + const loaded = persist ? loadEditorWindowState(app.getPath("userData")) : null; + const saved = loaded + ? { + ...clampRectToWorkArea(loaded, screen.getDisplayMatching(loaded).workArea), + maximized: loaded.maximized, + } + : null; + const creation = resolveEditorCreation({ isBench: query.windowType === "bench", saved }); const win = new BrowserWindow({ - width: 1200, - height: 800, + ...creation.bounds, minWidth: 800, minHeight: 600, // Seamless titlebar on every platform: the app's own topbar IS the titlebar @@ -425,7 +440,23 @@ export function createEditorWindow(query: Record = {}): BrowserW }, }); - win.maximize(); + if (creation.maximize) win.maximize(); + if (creation.persist) { + const persistState = () => { + if (win.isDestroyed()) return; + const bounds = win.getNormalBounds(); + saveEditorWindowState(app.getPath("userData"), { + x: bounds.x, + y: bounds.y, + width: bounds.width, + height: bounds.height, + maximized: win.isMaximized(), + }); + }; + win.on("moved", persistState); + win.on("resized", persistState); + win.on("close", persistState); + } // The editor renders its own File/Edit/View menu bar in the custom titlebar, // so hide the native OS menu bar on Windows/Linux (it stays reachable via Alt). diff --git a/flake.lock b/flake.lock index 77972fb40..dac7a986d 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1775710090, - "narHash": "sha256-ar3rofg+awPB8QXDaFJhJ2jJhu+KqN/PRCXeyuXR76E=", + "lastModified": 1788039129, + "narHash": "sha256-pa4Q0qErvCvzCaaUph7Sm37RhR4xvPrYI8Lgz6k85+A=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4c1018dae018162ec878d42fec712642d214fdfa", + "rev": "d2f67949798825fe853f7c5d0492b8bf016d3f88", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index be01a58b4..4a8c39f2f 100644 --- a/flake.nix +++ b/flake.nix @@ -2,6 +2,17 @@ description = "OpenScreen — desktop screen recorder with built-in editor"; inputs = { + # Do not roll flake.lock BACK past nixpkgs d2f6794 (2026-08-29). Before it, + # `importCargoLock` fetched every crate from + # `https://crates.io/api/v1/crates///download`, which crates.io + # now answers with 403 — it rate-limits that endpoint to 1 req/s and points + # clients at the CDN instead (rust-lang/crates.io#13482). Every crate in the + # lockfile failed, so `nix build` died in `cargo-vendor-dir` before reaching a + # single derivation of ours: `Nix build` was red on main from 2026-08-30, and + # since `nix-check.yml` only compares npmDepsHash and `nix-build.yml` did not + # run on pull requests, the derivation itself was not being built anywhere -- + # not before a merge, and not after one either while this was red. + # d2f6794 carries the switch to `https://static.crates.io/crates`. nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; }; diff --git a/nix/compositor-view.nix b/nix/compositor-view.nix index ecf8b0770..0b42a1bcd 100644 --- a/nix/compositor-view.nix +++ b/nix/compositor-view.nix @@ -120,7 +120,18 @@ rustPlatform.buildRustPackage { # Copy each library under its soname and read its symbol table. awk rather # than sed with a backreference: the third field is the name, and anything # after an @ is the version tag. - for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,avcodec,avutil,swscale,swresample}.so.*; do + # + # The list must hold every library crates/compositor/build.rs emits a + # cargo:rustc-link-lib for -- all six of them, avfilter included since the + # speed-region stretch runs through atempo. Miss one and the build either + # dies on `cannot find -lavfilter` (no unversioned symlink is staged for it + # below) or links the store's UN-renamed copy, and a cdylib tolerating + # undefined symbols means the failure surfaces only at require() time as + # "undefined symbol: osff_avfilter_graph_alloc" -- the addon then loads as a + # no-op and preview plus every export are dead. avfilter's exports are all + # av-prefixed (avfilter_*, av_buffersrc_*, av_buffersink_*), so the awk + # filter here and the leak check in installPhase already cover them. + for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,avcodec,avutil,swscale,swresample,avfilter}.so.*; do case "$lib" in *.so.*.*) continue ;; esac test -f "$lib" || continue cp "$(readlink -f "$lib")" "$stage/lib/$(basename "$lib")" @@ -178,7 +189,7 @@ rustPlatform.buildRustPackage { # Each copy still carries the RUNPATH it inherited from the original ffmpeg # output, which is where the UN-renamed libraries live -- so libavcodec's own # osff_swr_init would resolve against a libswresample that defines swr_init. - # It only works today because all five happen to be direct DT_NEEDED of the + # It only works today because all six happen to be direct DT_NEEDED of the # addon, so $ORIGIN is searched first; the day --as-needed drops one the # loader falls through to the store copy and dlopen fails on an undefined # osff_ symbol. Put $ORIGIN in front so the renamed set can only resolve diff --git a/nix/package.nix b/nix/package.nix index 9e5dd1e23..02e9eb10a 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -51,7 +51,7 @@ buildNpmPackage { ); }; - npmDepsHash = "sha256-Vr6Sw/WKmX22eT4a22+Xr3/miMzZr2uAwiYx12toU/E="; + npmDepsHash = "sha256-LkKX1edTPHZq5nQRrbLAn11oVw36kb0smNQMmVRMEPA="; env.ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; diff --git a/nix/pipewire-helper.nix b/nix/pipewire-helper.nix index 953171c94..61268397a 100644 --- a/nix/pipewire-helper.nix +++ b/nix/pipewire-helper.nix @@ -22,6 +22,7 @@ pkg-config, patchelfUnstable, pipewire, + libglvnd, }: let @@ -92,6 +93,17 @@ rustPlatform.buildRustPackage { # loader in compositor-view.nix: an soname reached by dlopen is invisible to the # linker and has to be added deliberately. # + # libglvnd is here for the same reason and needs no build dependency either: + # csrc/dmabuf_modifiers.c spells out the handful of EGL types it uses rather + # than including , and reaches the entry points through + # `dlopen("libEGL.so.1")` + dlsym. Without the RPATH entry that dlopen fails, + # the modifier query returns nothing, and the format offer degrades to + # LINEAR/INVALID -- so a tiled compositor buffer, the common case on + # AMD/mutter, negotiates as shm instead of dmabuf and the zero-copy path is + # lost with nothing in any log to say so. libglvnd is the dispatch library + # only; the vendor ICD stays the host's job via /run/opengl-driver, exactly as + # the Vulkan loader leaves it in compositor-view.nix. + # # --force-rpath because build.rs passes -Wl,--disable-new-dtags on purpose: it # wants DT_RPATH rather than DT_RUNPATH, so that the entries apply to the # transitive ffmpeg libraries too. patchelf defaults to writing DT_RUNPATH, @@ -111,7 +123,12 @@ rustPlatform.buildRustPackage { # entry survives. dontPatchELF would also work and is worse: it disables the # shrink for every output, to fix one entry. postFixup = '' - patchelf --force-rpath --add-rpath "${lib.makeLibraryPath [ pipewire ]}" \ + patchelf --force-rpath --add-rpath "${ + lib.makeLibraryPath [ + pipewire + libglvnd + ] + }" \ "$out/bin/openscreen-pipewire-helper" ''; diff --git a/package-lock.json b/package-lock.json index e0ea6e09c..364c07429 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,13 @@ { "name": "openscreen", - "version": "1.9.6", + "version": "1.10.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openscreen", - "version": "1.9.6", + "version": "1.10.0", + "license": "MIT", "dependencies": { "@fix-webm-duration/fix": "^1.0.1", "@langchain/anthropic": "^1.3.26", @@ -66,7 +67,7 @@ "@types/react-dom": "^18.3.7", "@vitejs/plugin-react": "^5.2.0", "autoprefixer": "^10.5.0", - "electron": "^41.2.1", + "electron": "41.2.1", "electron-builder": "^26.15.3", "esbuild": "^0.28.1", "fast-check": "^4.7.0", diff --git a/package.json b/package.json index 412f6dfb2..8a630d3b5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "openscreen", "private": true, - "version": "1.9.6", + "version": "1.10.0", "description": "Record your screen and polish the demo", "homepage": "https://getopenscreen.com/", "license": "MIT", @@ -42,16 +42,16 @@ "assets:appx": "node scripts/generate-appx-assets.mjs", "preview": "vite preview", "build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs", - "build:mac": "npm run build:native:mac && npm run fetch:ffmpeg:mac && npm run build:native:compositor:mac && tsc && vite build && electron-builder --mac", + "build:mac": "npm run build:native:mac && npm run fetch:ffmpeg:mac && npm run fetch:onnxruntime && npm run build:native:compositor:mac && tsc && vite build && electron-builder --mac", "build:native:win": "node scripts/build-windows-wgc-helper.mjs", "stage:vcomp": "node scripts/stage-vcomp-runtime.mjs", "build:native:compositor": "node scripts/build-windows-compositor-addon.mjs", "build:native:compositor:mac": "node scripts/build-macos-compositor-addon.mjs", "build:native:compositor:linux": "node scripts/build-linux-compositor-addon.mjs", "build:native:linux": "node scripts/build-linux-pipewire-helper.mjs", - "build:win": "npm run build:native:win && npm run fetch:ffmpeg && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win --config.npmRebuild=false", - "build:win:store": "npm run build:native:win && npm run fetch:ffmpeg && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win appx --config.npmRebuild=false", - "build:linux": "npm run fetch:ffmpeg:sdk && npm run build:native:linux && npm run build:native:compositor:linux && tsc && vite build && electron-builder --linux AppImage deb pacman rpm --config.npmRebuild=false", + "build:win": "npm run build:native:win && npm run fetch:ffmpeg && npm run fetch:onnxruntime && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win --config.npmRebuild=false", + "build:win:store": "npm run build:native:win && npm run fetch:ffmpeg && npm run fetch:onnxruntime && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win appx --config.npmRebuild=false", + "build:linux": "npm run fetch:ffmpeg:sdk && npm run build:native:linux && npm run fetch:onnxruntime && npm run build:native:compositor:linux && tsc && vite build && electron-builder --linux AppImage deb pacman rpm --config.npmRebuild=false", "build:whisper-binaries": "bash scripts/build-whisper-stt.sh", "test:whisper-stt": "node scripts/test-whisper-stt.mjs", "test": "vitest --run", @@ -92,7 +92,8 @@ "prepare": "husky", "fetch:ffmpeg": "node scripts/fetch-ffmpeg.mjs", "fetch:ffmpeg:mac": "node scripts/fetch-ffmpeg-macos.mjs", - "fetch:ffmpeg:sdk": "node scripts/fetch-ffmpeg.mjs --sdk-only" + "fetch:ffmpeg:sdk": "node scripts/fetch-ffmpeg.mjs --sdk-only", + "fetch:onnxruntime": "node scripts/fetch-onnxruntime.mjs" }, "dependencies": { "@fix-webm-duration/fix": "^1.0.1", @@ -153,7 +154,7 @@ "@types/react-dom": "^18.3.7", "@vitejs/plugin-react": "^5.2.0", "autoprefixer": "^10.5.0", - "electron": "^41.2.1", + "electron": "41.2.1", "electron-builder": "^26.15.3", "esbuild": "^0.28.1", "fast-check": "^4.7.0", diff --git a/public/mediapipe/selfie_segmentation/README.md b/public/mediapipe/selfie_segmentation/README.md new file mode 100644 index 000000000..f39999be2 --- /dev/null +++ b/public/mediapipe/selfie_segmentation/README.md @@ -0,0 +1,46 @@ +# MediaPipe Selfie Segmentation — model weights + +Only the model weights live here. The MediaPipe **JavaScript** solution (the `.js` glue and the +two ~5.6 MB `.wasm` builds) was removed when segmentation moved into the native compositor: the +renderer no longer runs inference at all, so nothing loaded them. + +Upstream: , Apache-2.0. + +The `.tflite` files are kept because the `.onnx` below is **derived from them** — they are the +provenance, not dead weight. + +## `selfie_segmentation_landscape.onnx` — derived, not vendored + +The `.onnx` beside the `.tflite` files is **generated from them**, by +[`scripts/convert-selfie-segmentation-to-onnx.py`](../../../scripts/convert-selfie-segmentation-to-onnx.py). +No weights were downloaded; it is a derived work of the MediaPipe model already vendored here +(Apache-2.0). + +It exists because the realtime path runs inference through ONNX Runtime rather than the +MediaPipe JS solution. Regenerate with: + +``` +pip install "numpy<2" "tensorflow==2.13.1" "tf2onnx==1.16.1" "onnx==1.16.2" "protobuf<4" +python scripts/convert-selfie-segmentation-to-onnx.py landscape +``` + +**The conversion is not mechanical.** `tf2onnx` exits 0 while leaving 12 operators that ONNX +Runtime cannot load — 11 `HardSwish` emitted into an opset-13 graph, and MediaPipe's custom +`TFL_Convolution2DTransposeBias`. The script repairs both; the reasoning is in its docstring. +If you regenerate, re-check the mask on a real frame rather than trusting the exit code. + +| | | +|---|---| +| input | `input_1` `[1, 144, 256, 3]` float32, **NHWC**, RGB scaled to 0..1 | +| output | `segment_back` `[1, 144, 256, 1]` float32, already sigmoid-activated | +| feeds | the compositor's `t3` mask slot (256x144 R8) | + +The graph is fully convolutional and resolution-agnostic, so the input dimensions can be +rewritten in place — but **both dimensions must be divisible by 16**, or the skip-connection +`Add`s fail on mismatched extents. Measured quality below 192x112 degrades visibly on a +full-screen camera, and at 64x48 the model stops producing a mask at all. + +> **Packaging:** this file currently sits under `public/`, which is bundled into `app.asar`. +> Anything that resolves a filesystem path for native code cannot read it from there — see +> `scripts/before-pack.cjs` and the compositor's asset handling. Whoever wires the loader +> should decide whether it moves to `extraResources` or is read through the renderer. diff --git a/public/mediapipe/selfie_segmentation/selfie_segmentation.tflite b/public/mediapipe/selfie_segmentation/selfie_segmentation.tflite new file mode 100644 index 000000000..374c0720d Binary files /dev/null and b/public/mediapipe/selfie_segmentation/selfie_segmentation.tflite differ diff --git a/public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx b/public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx new file mode 100644 index 000000000..1fdedb18c Binary files /dev/null and b/public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.onnx differ diff --git a/public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.tflite b/public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.tflite new file mode 100644 index 000000000..4ea3f8a10 Binary files /dev/null and b/public/mediapipe/selfie_segmentation/selfie_segmentation_landscape.tflite differ diff --git a/public/wallpapers/wallpaper1.jpg b/public/wallpapers/wallpaper1.jpg index dbd8afb8b..08d065b8c 100644 Binary files a/public/wallpapers/wallpaper1.jpg and b/public/wallpapers/wallpaper1.jpg differ diff --git a/public/wallpapers/wallpaper10.jpg b/public/wallpapers/wallpaper10.jpg index 49da791cb..eebff5b57 100644 Binary files a/public/wallpapers/wallpaper10.jpg and b/public/wallpapers/wallpaper10.jpg differ diff --git a/public/wallpapers/wallpaper11.jpg b/public/wallpapers/wallpaper11.jpg index b20004068..189a7fdea 100644 Binary files a/public/wallpapers/wallpaper11.jpg and b/public/wallpapers/wallpaper11.jpg differ diff --git a/public/wallpapers/wallpaper12.jpg b/public/wallpapers/wallpaper12.jpg index 264357dd5..d4fc0f6ee 100644 Binary files a/public/wallpapers/wallpaper12.jpg and b/public/wallpapers/wallpaper12.jpg differ diff --git a/public/wallpapers/wallpaper13.jpg b/public/wallpapers/wallpaper13.jpg index 052f0ff7f..76dda6762 100644 Binary files a/public/wallpapers/wallpaper13.jpg and b/public/wallpapers/wallpaper13.jpg differ diff --git a/public/wallpapers/wallpaper14.jpg b/public/wallpapers/wallpaper14.jpg index 233238a45..ab793307f 100644 Binary files a/public/wallpapers/wallpaper14.jpg and b/public/wallpapers/wallpaper14.jpg differ diff --git a/public/wallpapers/wallpaper15.jpg b/public/wallpapers/wallpaper15.jpg index 5742aab45..adf12c100 100644 Binary files a/public/wallpapers/wallpaper15.jpg and b/public/wallpapers/wallpaper15.jpg differ diff --git a/public/wallpapers/wallpaper16.jpg b/public/wallpapers/wallpaper16.jpg index 1f9efcea1..8393d5759 100644 Binary files a/public/wallpapers/wallpaper16.jpg and b/public/wallpapers/wallpaper16.jpg differ diff --git a/public/wallpapers/wallpaper17.jpg b/public/wallpapers/wallpaper17.jpg index d7188a473..962af6dbd 100644 Binary files a/public/wallpapers/wallpaper17.jpg and b/public/wallpapers/wallpaper17.jpg differ diff --git a/public/wallpapers/wallpaper18.jpg b/public/wallpapers/wallpaper18.jpg index 9976c8cef..dc097b92b 100644 Binary files a/public/wallpapers/wallpaper18.jpg and b/public/wallpapers/wallpaper18.jpg differ diff --git a/public/wallpapers/wallpaper3.jpg b/public/wallpapers/wallpaper3.jpg index 73d60c55d..fc96b9327 100644 Binary files a/public/wallpapers/wallpaper3.jpg and b/public/wallpapers/wallpaper3.jpg differ diff --git a/public/wallpapers/wallpaper4.jpg b/public/wallpapers/wallpaper4.jpg index 49f281643..e85c00565 100644 Binary files a/public/wallpapers/wallpaper4.jpg and b/public/wallpapers/wallpaper4.jpg differ diff --git a/public/wallpapers/wallpaper5.jpg b/public/wallpapers/wallpaper5.jpg index 294ca329c..95ce44af3 100644 Binary files a/public/wallpapers/wallpaper5.jpg and b/public/wallpapers/wallpaper5.jpg differ diff --git a/public/wallpapers/wallpaper6.jpg b/public/wallpapers/wallpaper6.jpg index 419569d68..a69b74391 100644 Binary files a/public/wallpapers/wallpaper6.jpg and b/public/wallpapers/wallpaper6.jpg differ diff --git a/public/wallpapers/wallpaper7.jpg b/public/wallpapers/wallpaper7.jpg index aa9d818d8..c59d8eebc 100644 Binary files a/public/wallpapers/wallpaper7.jpg and b/public/wallpapers/wallpaper7.jpg differ diff --git a/public/wallpapers/wallpaper8.jpg b/public/wallpapers/wallpaper8.jpg index 14e5ca595..21992ae98 100644 Binary files a/public/wallpapers/wallpaper8.jpg and b/public/wallpapers/wallpaper8.jpg differ diff --git a/public/wallpapers/wallpaper9.jpg b/public/wallpapers/wallpaper9.jpg index be273b486..b574c378c 100644 Binary files a/public/wallpapers/wallpaper9.jpg and b/public/wallpapers/wallpaper9.jpg differ diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index 70a2d6ccb..f92d8e7ed 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -68,6 +68,26 @@ const HELPER_SOURCE_PATHS = [ * `mac.extraResources` ships this directory wholesale (`filter: ["darwin-*​/*"]`), so * "present here" is the same thing as "present in the installed app". */ +/** + * L'exigence ONNX Runtime de macOS, séparée parce qu'elle ne vaut QUE sur arm64. + * + * L'amont ne publie aucun binaire ONNX pour les Macs Intel : `fetch-onnxruntime.mjs` le constate + * et sort en 0 sans rien poser. Un paquet x64 sans la bibliothèque est donc CORRECT, et l'exiger + * là ferait échouer à l'empaquetage une build parfaitement saine — la garde se retournerait + * contre ce qu'elle protège. + * + * Sur arm64 en revanche son absence ne casse rien de visible : `Segmenter::load` refuse, le + * compositeur dessine la webcam telle quelle, et le contrôle disparaît de l'éditeur. Le paquet + * est silencieusement amputé, ce qui est exactement la panne que cette garde existe pour + * attraper et qu'aucun test ne peut voir puisque tout se dégrade proprement. + */ +const MAC_ONNX_REQUIRED = { + match: (name) => name === "libonnxruntime.dylib", + what: "the ONNX Runtime library the camera-background segmentation loads", + breaks: "the camera-background control vanishes from the editor and every effect is a no-op", + fix: "Stage it with:\n\n npm run fetch:onnxruntime", +}; + const MAC_REQUIRED = [ { match: (name) => name === "compositor_view.node", @@ -75,12 +95,21 @@ const MAC_REQUIRED = [ breaks: "the preview and every export render nothing", fix: FIX_MAC, }, - { - match: (name) => /^libav(codec|format|util)\.\d+\.dylib$/.test(name), - what: "the LGPL ffmpeg dylibs the compositor links", + // One requirement per library, not `atLeast: N` over a combined regex — the + // same trap LINUX_REQUIRED documents above. Several versioned copies of one + // library would satisfy a combined count while another was missing entirely, + // and the addon would still fail to load. + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ + match: (name) => new RegExp(`^lib${library}\\.\\d+\\.dylib$`).test(name), + what: `the LGPL lib${library} dylib the compositor links`, breaks: "the compositor addon cannot be loaded at all (dyld error at require())", fix: FIX_MAC, - atLeast: 3, + })), + { + match: (name) => /^libavdevice\.\d+\.dylib$/.test(name), + what: "the LGPL libavdevice dylib the ffmpeg CLI links", + breaks: "ffmpeg dies in dyld before main(), so waveform and STT extraction cannot start", + fix: FIX_MAC, }, { match: (name) => name === "whisper-stt-server", @@ -101,6 +130,14 @@ const MAC_REQUIRED = [ breaks: "native screen capture is unavailable", fix: "Build it with:\n\n npm run build:native:mac", }, + { + match: (name) => name === "ffmpeg", + what: "the LGPL ffmpeg CLI (spawned for waveform peaks and STT audio extraction)", + breaks: + "transcription falls back to the renderer decode or fails outright on machines with no\n" + + 'system ffmpeg, shown to the user only as "Failed to fetch" (#616)', + fix: "Build it with:\n\n npm run build:native:compositor:mac\n\nwhich stages the SDK's ffmpeg beside the vendored dylibs.", + }, ]; /** @@ -119,6 +156,17 @@ const MAC_REQUIRED = [ * `helper-ffmpeg/` subdirectory holds. */ const LINUX_REQUIRED = [ + // L'effet de fond de caméra est le seul dont la présence dépend d'un binaire optionnel, et + // son absence ne casse RIEN de visible : `Segmenter::load` refuse, le compositeur dessine la + // webcam telle quelle, et le contrôle disparaît de l'éditeur. Un paquet livré sans elle est + // donc silencieusement amputé — la panne exacte que cette garde existe pour attraper, et + // celle qu'aucun test ne peut voir puisque tout se dégrade proprement. + { + match: (name) => name === "libonnxruntime.so", + what: "the ONNX Runtime library the camera-background segmentation loads", + breaks: "the camera-background control vanishes from the editor and every effect is a no-op", + fix: "Stage it with:\n\n npm run fetch:onnxruntime", + }, { match: (name) => name === "compositor_view.node", what: "the wgpu/Vulkan compositor addon", @@ -131,7 +179,7 @@ const LINUX_REQUIRED = [ // pendant qu'une autre manquait. Le paquet passait alors la garde et le // compositeur ne chargeait pas : exactement le mode de panne que cette garde // existe pour attraper. - ...["avcodec", "avformat", "avutil", "swresample", "swscale"].map((library) => ({ + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ match: (name) => new RegExp(`^lib${library}\\.so\\.\\d+$`).test(name), what: `the symbol-renamed lib${library} shared object the compositor links`, breaks: "the compositor addon cannot be loaded at all (ld.so error at require())", @@ -223,6 +271,17 @@ function checkNativePayload({ dir, required, osLabel, bundleNoun, emptyDirFix }) * "together here" is the same thing as "together in the installed app". */ const WIN_REQUIRED = [ + // L'effet de fond de caméra est le seul dont la présence dépend d'un binaire optionnel, et + // son absence ne casse RIEN de visible : `Segmenter::load` refuse, le compositeur dessine la + // webcam telle quelle, et le contrôle disparaît de l'éditeur. Un paquet livré sans elle est + // donc silencieusement amputé — la panne exacte que cette garde existe pour attraper, et + // celle qu'aucun test ne peut voir puisque tout se dégrade proprement. + { + match: (name) => name === "onnxruntime.dll", + what: "the ONNX Runtime library the camera-background segmentation loads", + breaks: "the camera-background control vanishes from the editor and every effect is a no-op", + fix: "Stage it with:\n\n npm run fetch:onnxruntime", + }, { match: (name) => name === "compositor_view.node", what: "the D3D11 compositor addon", @@ -234,7 +293,7 @@ const WIN_REQUIRED = [ // (avcodec-60/61/62.dll left by an earlier fetch) would satisfy a combined count // while another library was missing entirely, and the addon would still fail to // load. - ...["avcodec", "avformat", "avutil"].map((library) => ({ + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ match: (name) => new RegExp(`^${library}-\\d+\\.dll$`).test(name), what: `the ${library} DLL the compositor links`, breaks: "the addon cannot be loaded at all under MSIX, which ignores PATH", @@ -419,13 +478,22 @@ function checkWinNativePayload() { } function checkMacNativePayload(context) { + const arch = archTagFor(context); + const dir = path.join(ROOT, "electron", "native", "bin", `darwin-${arch}`); checkNativePayload({ - dir: path.join(ROOT, "electron", "native", "bin", `darwin-${archTagFor(context)}`), - required: MAC_REQUIRED, + dir, + // Voir `MAC_ONNX_REQUIRED` : exiger la bibliothèque sur Intel ferait échouer une build + // que l'amont rend impossible à satisfaire. + required: arch === "arm64" ? [...MAC_REQUIRED, MAC_ONNX_REQUIRED] : MAC_REQUIRED, osLabel: "macOS", bundleNoun: "the .app", emptyDirFix: `${FIX_MAC}\n\nThe STT helper and the capture helper are separate builds — see\ntechnical-documentation/engineering/build-and-packaging.md.`, }); + + // "Complete" is not the same property as "runnable on the macOS we claim". This file + // exists because a payload can be whole and still broken; a floor above the supported + // one is the second way that happens. See checkMacOsVersionFloor(). + checkMacOsVersionFloor(dir); } function checkLinuxNativePayload(context) { @@ -504,6 +572,18 @@ function checkLinuxNativePayload(context) { */ const MAX_SYMBOL_VERSION = { GLIBC: "2.35", GLIBCXX: "3.4.30", CXXABI: "1.3.13" }; +/** + * The oldest macOS anything in the payload may demand — the macOS twin of + * MAX_SYMBOL_VERSION above, and the same class of bug on a different libc. + * + * Must equal `mac.minimumSystemVersion` in electron-builder.json5, which is what the .app + * tells LaunchServices; before-pack.test.mjs asserts exactly that, so the two cannot drift + * apart quietly. Not read from the config at runtime because this hook must keep working + * if that file is ever restructured — a guard that throws while parsing is a guard that + * gets deleted. + */ +const MAC_MIN_OS_FLOOR = "13.0"; + /** * The one supported way past the ceiling, for the one case it does not fit: a developer * on a distro newer than the floor, building a package for their own machine. @@ -718,7 +798,15 @@ function resolveSymbolCeiling() { // are the only things standing between this escape hatch and a published package that // starts on nobody's machine but the builder's, and they are reachable from a test // without a payload to scan — so they are tested rather than trusted. -exports.__testing = { resolveSymbolCeiling, MAX_SYMBOL_VERSION }; +exports.__testing = { + resolveSymbolCeiling, + MAX_SYMBOL_VERSION, + machoMinOs, + checkMacOsVersionFloor, + MAC_MIN_OS_FLOOR, + MAC_REQUIRED, + checkNativePayload, +}; /** Every ELF under `dir`, recursively — the helper's ffmpeg sits in a subdirectory. */ function elfFilesUnder(dir) { @@ -820,6 +908,158 @@ function checkLinuxSymbolVersionFloor(dir) { ); } +/** + * The macOS minimum-OS a Mach-O declares, as "12.0", or null if it declares none. + * + * Reads LC_BUILD_VERSION (and LC_VERSION_MIN_MACOSX, which is what anything built + * against an older SDK carries) straight out of the file. Parsed here rather than + * shelled out to `vtool -show-build` for the same reason neededSymbolVersions() does not + * use readelf and importedDlls() does not use dumpbin — but with an extra one on top: + * this hook runs for the Windows and Linux packs too, and vtool exists on neither, so a + * subprocess would have to be skipped on exactly the hosts where skipping is silent. + * Parsing makes the guard host-independent instead of conditionally absent. + * + * Universal binaries are walked slice by slice and the HIGHEST floor wins: an x86_64 half + * built on a newer machine strands Intel users just as thoroughly as a thin binary would. + */ +function machoMinOs(file) { + const b = fs.readFileSync(file); + const FAT_MAGIC = 0xcafebabe; + const FAT_MAGIC_64 = 0xcafebabf; + const MH_MAGIC_64 = 0xfeedfacf; + const MH_MAGIC_32 = 0xfeedface; + const LC_VERSION_MIN_MACOSX = 0x24; + const LC_BUILD_VERSION = 0x32; + const PLATFORM_MACOS = 1; + + /** X.Y.Z packed as nibbles: 0x000c0000 is 12.0.0. */ + const decode = (packed) => `${packed >>> 16}.${(packed >> 8) & 0xff}.${packed & 0xff}`; + + const sliceMinOs = (start) => { + const magic = b.readUInt32LE(start); + if (magic !== MH_MAGIC_64 && magic !== MH_MAGIC_32) return null; + const ncmds = b.readUInt32LE(start + 16); + // 32 bytes of mach_header_64 (28 + 4 bytes of `reserved`); 28 for the 32-bit one. + let off = start + (magic === MH_MAGIC_64 ? 32 : 28); + for (let i = 0; i < ncmds; i++) { + if (off + 8 > b.length) return null; + const cmd = b.readUInt32LE(off); + const cmdsize = b.readUInt32LE(off + 4); + if (cmdsize < 8) return null; + if (cmd === LC_BUILD_VERSION && b.readUInt32LE(off + 8) === PLATFORM_MACOS) { + return decode(b.readUInt32LE(off + 12)); + } + if (cmd === LC_VERSION_MIN_MACOSX) { + return decode(b.readUInt32LE(off + 8)); + } + off += cmdsize; + } + return null; + }; + + const fat = b.readUInt32BE(0); + if (fat === FAT_MAGIC || fat === FAT_MAGIC_64) { + const wide = fat === FAT_MAGIC_64; + const nfat = b.readUInt32BE(4); + let best = null; + for (let i = 0; i < nfat; i++) { + const entry = 8 + i * (wide ? 32 : 20); + const offset = wide ? Number(b.readBigUInt64BE(entry + 8)) : b.readUInt32BE(entry + 8); + const found = sliceMinOs(offset); + if (found && (!best || compareVersions(found, best) > 0)) best = found; + } + return best; + } + + return sliceMinOs(0); +} + +/** Every Mach-O under `dir`, recursively. Symlinks are skipped — see elfFilesUnder(). */ +function machoFilesUnder(dir) { + const found = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + found.push(...machoFilesUnder(full)); + continue; + } + if (!entry.isFile()) continue; + // By magic, not by extension: the helpers and whisper-stt-server have none, and + // the ggml/whisper dylibs come as chains of symlinks onto one real file. + const magic = Buffer.alloc(4); + const fd = fs.openSync(full, "r"); + try { + fs.readSync(fd, magic, 0, 4, 0); + } finally { + fs.closeSync(fd); + } + const le = magic.readUInt32LE(0); + const be = magic.readUInt32BE(0); + if (le === 0xfeedfacf || le === 0xfeedface || be === 0xcafebabe || be === 0xcafebabf) { + found.push(full); + } + } + return found; +} + +/** Nothing we ship may demand a newer macOS than MAC_MIN_OS_FLOOR. */ +function checkMacOsVersionFloor(dir) { + const scanned = machoFilesUnder(dir).map((file) => ({ + name: path.relative(dir, file), + minOs: machoMinOs(file), + })); + + // Same assertion the Linux floor makes, for the same reason: a guard that quietly + // stops looking reports "clean" for the rest of the project's life. Every binary we + // ship is built with a deployment target, so reading none from any of them means the + // parser broke rather than that the payload is unusually clean. + if (scanned.length > 0 && !scanned.some((entry) => entry.minOs)) { + throw new Error( + `Refusing to package: read no macOS deployment target from any of the ${scanned.length} ` + + `Mach-O files in ${path.relative(ROOT, dir)}.\n\n` + + "Every one of them carries LC_BUILD_VERSION, so this is a bug in machoMinOs()\n" + + "(scripts/before-pack.cjs), not an unusually clean payload. Fix the parser — leaving\n" + + "it is how a build that cannot start on the supported macOS gets shipped again.", + ); + } + + const offenders = scanned.filter( + (entry) => entry.minOs && compareVersions(entry.minOs, MAC_MIN_OS_FLOOR) > 0, + ); + if (offenders.length === 0) { + return; + } + + throw new Error( + `Refusing to package binaries that demand a newer macOS than the ${MAC_MIN_OS_FLOOR} floor\n` + + "the app claims to support.\n\n" + + ` looked in: ${path.relative(ROOT, dir)}\n\n` + + `${offenders.map((o) => ` - ${o.name} is built for macOS ${o.minOs} (floor ${MAC_MIN_OS_FLOOR})`).join("\n")}\n\n` + + "Almost certainly nothing asked for this: clang and CMake default the deployment\n" + + "target to the BUILD MACHINE's SDK, so this usually means a build script forgot to\n" + + "pin one and the floor followed whatever image compiled it. CI's macos-latest moves\n" + + "on its own, so the same source can ship a different floor month to month.\n\n" + + "The number itself is not what breaks: dyld does NOT refuse a binary whose minos\n" + + "exceeds the running OS. The damage is done at link time — the deployment target\n" + + "decides which symbols the linker resolves against the OS instead of emitting\n" + + "locally, so a too-high floor leaves strong references to symbols the target macOS\n" + + "has never had, and the binary dies in dyld with 'Symbol not found'. That is issue\n" + + "#515: a helper built for 13 stranded every macOS 12 user, and the app reported it\n" + + "as a denied Accessibility permission.\n\n" + + "Pin the deployment target in whichever script built the file:\n\n" + + // Derived, not spelled out: this line said `.v12` for a while after the floor + // moved to 13, i.e. the guard's own remediation advice contradicted the floor + // it was enforcing. + ` Swift platforms: [.macOS(.v${MAC_MIN_OS_FLOOR.split(".")[0]})] electron/native/screencapturekit/Package.swift\n` + + " CMake -DCMAKE_OSX_DEPLOYMENT_TARGET scripts/build-whisper-stt.sh\n" + + " clang -mmacosx-version-min scripts/fetch-ffmpeg-macos.mjs\n" + + " rustc MACOSX_DEPLOYMENT_TARGET scripts/build-macos-compositor-addon.mjs\n\n" + + "To see it yourself:\n\n" + + " vtool -show-build \n\n" + + `Raising MAC_MIN_OS_FLOOR drops a macOS version the README claims to support.`, + ); +} + /** Newest mtime under `target` (file or directory), or 0 if it does not exist. */ function newestMtimeMs(target) { let stat; diff --git a/scripts/before-pack.test.mjs b/scripts/before-pack.test.mjs index 905283510..a0c356894 100644 --- a/scripts/before-pack.test.mjs +++ b/scripts/before-pack.test.mjs @@ -88,3 +88,272 @@ describe("symbol-version ceiling", () => { }); }); }); + +// --------------------------------------------------------------------------- +// macOS deployment floor (issue #515) +// --------------------------------------------------------------------------- +// +// Mach-O headers are synthesised here rather than compiled with clang, so this runs on +// the Linux and Windows CI legs too. That is the same reason the guard parses the file +// itself instead of shelling out to `vtool`: the check has to be present everywhere the +// hook is, not conditionally absent on the hosts where nobody would notice. +// +// The parser is separately cross-checked against the real thing — on a machine with a +// staged macOS payload, every Mach-O in it agreed with `vtool -show-build` (44/44). + +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { declaredAppVersionFrom } from "./macos-floor.mjs"; + +/** Packs X.Y.Z the way LC_BUILD_VERSION does: one byte per component, X in the top half. */ +function packVersion(text) { + const [x = 0, y = 0, z = 0] = text.split(".").map(Number); + return ((x & 0xffff) << 16) | ((y & 0xff) << 8) | (z & 0xff); +} + +/** A 64-bit Mach-O carrying exactly one load command: LC_BUILD_VERSION for macOS. */ +function thinMachO(minOs) { + const header = Buffer.alloc(32); + header.writeUInt32LE(0xfeedfacf, 0); // MH_MAGIC_64 + header.writeUInt32LE(1, 16); // ncmds + const lc = Buffer.alloc(24); + lc.writeUInt32LE(0x32, 0); // LC_BUILD_VERSION + lc.writeUInt32LE(24, 4); // cmdsize + lc.writeUInt32LE(1, 8); // PLATFORM_MACOS + lc.writeUInt32LE(packVersion(minOs), 12); + return Buffer.concat([header, lc]); +} + +/** The older spelling, which anything built against an older SDK carries instead. */ +function thinMachOVersionMin(minOs) { + const header = Buffer.alloc(32); + header.writeUInt32LE(0xfeedfacf, 0); + header.writeUInt32LE(1, 16); + const lc = Buffer.alloc(16); + lc.writeUInt32LE(0x24, 0); // LC_VERSION_MIN_MACOSX + lc.writeUInt32LE(16, 4); + lc.writeUInt32LE(packVersion(minOs), 8); + return Buffer.concat([header, lc]); +} + +/** A universal binary whose slices disagree — the highest floor is the one that counts. */ +function fatMachO(minOsPerSlice) { + const headerSize = 8 + minOsPerSlice.length * 20; + const head = Buffer.alloc(headerSize); + head.writeUInt32BE(0xcafebabe, 0); + head.writeUInt32BE(minOsPerSlice.length, 4); + const slices = minOsPerSlice.map(thinMachO); + let offset = headerSize; + slices.forEach((slice, i) => { + const entry = 8 + i * 20; + head.writeUInt32BE(offset, entry + 8); // offset + head.writeUInt32BE(slice.length, entry + 12); // size + offset += slice.length; + }); + return Buffer.concat([head, ...slices]); +} + +function withPayload(files, body) { + const dir = mkdtempSync(path.join(tmpdir(), "openscreen-minos-")); + try { + for (const [name, bytes] of Object.entries(files)) { + writeFileSync(path.join(dir, name), bytes); + } + return body(dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const testing = () => require(BEFORE_PACK).__testing; + +describe("machoMinOs", () => { + it("reads LC_BUILD_VERSION", () => { + withPayload({ helper: thinMachO("12.0") }, (dir) => { + expect(testing().machoMinOs(path.join(dir, "helper"))).toBe("12.0.0"); + }); + }); + + it("reads the older LC_VERSION_MIN_MACOSX spelling", () => { + withPayload({ helper: thinMachOVersionMin("11.3") }, (dir) => { + expect(testing().machoMinOs(path.join(dir, "helper"))).toBe("11.3.0"); + }); + }); + + it("takes the HIGHEST floor across a universal binary's slices", () => { + // An arm64 half built correctly does not rescue an x86_64 half that was not: + // Intel users are stranded just as thoroughly. + withPayload({ fat: fatMachO(["12.0", "26.0"]) }, (dir) => { + expect(testing().machoMinOs(path.join(dir, "fat"))).toBe("26.0.0"); + }); + }); + + it("returns null for a Mach-O that declares no deployment target", () => { + const header = Buffer.alloc(32); + header.writeUInt32LE(0xfeedfacf, 0); + withPayload({ bare: header }, (dir) => { + expect(testing().machoMinOs(path.join(dir, "bare"))).toBeNull(); + }); + }); +}); + +describe("checkMacOsVersionFloor", () => { + /** + * Fixtures are derived from the floor rather than written as literals. An earlier + * revision hardcoded the then-current floor as "the offending version", and raising + * the floor silently turned the offender into a compliant binary — the guard's own + * tests stopped testing it. The exact versions were never the point; being on the + * wrong side of the floor is. + */ + const floorMajor = Number(testing().MAC_MIN_OS_FLOOR.split(".")[0]); + const above = (bump = 1) => `${floorMajor + bump}.0`; + const below = () => `${floorMajor - 1}.0`; + + it("passes a payload built at or below the floor", () => { + withPayload({ a: thinMachO(testing().MAC_MIN_OS_FLOOR), b: thinMachO(below()) }, (dir) => { + expect(() => testing().checkMacOsVersionFloor(dir)).not.toThrow(); + }); + }); + + /** + * The regression test for #515: the helper that stranded Monterey was built for 13, + * and nothing in the pipeline looked. The message has to carry enough for whoever + * hits it to understand the consequence rather than just raise the constant. + */ + it("refuses a binary built above the floor, and says which and why", () => { + withPayload({ "openscreen-macos-cursor-helper": thinMachO(above()) }, (dir) => { + let message = ""; + try { + testing().checkMacOsVersionFloor(dir); + } catch (err) { + message = err.message; + } + expect(message).toContain("openscreen-macos-cursor-helper"); + expect(message).toContain(`macOS ${above()}.0`); + expect(message).toContain(testing().MAC_MIN_OS_FLOOR); + expect(message).toContain("#515"); + // The mechanism, so nobody "fixes" it by assuming dyld gates on the number. + expect(message).toContain("Symbol not found"); + }); + }); + + it("reports every offender, not just the first", () => { + withPayload( + { + ok: thinMachO(testing().MAC_MIN_OS_FLOOR), + bad1: thinMachO(above(1)), + bad2: thinMachO(above(2)), + }, + (dir) => { + expect(() => testing().checkMacOsVersionFloor(dir)).toThrow(/bad1[\s\S]*bad2/); + }, + ); + }); + + it("shouts if it parsed nothing, rather than reporting a clean payload", () => { + const header = Buffer.alloc(32); + header.writeUInt32LE(0xfeedfacf, 0); + withPayload({ bare: header }, (dir) => { + expect(() => testing().checkMacOsVersionFloor(dir)).toThrow(/bug in machoMinOs/); + }); + }); + + it("says nothing about a directory with no Mach-O in it", () => { + // Non-macOS packs reach this only if the tree exists; an empty one is not an error + // here — checkNativePayload already owns "the payload is incomplete". + withPayload({ "notes.txt": Buffer.from("hello") }, (dir) => { + expect(() => testing().checkMacOsVersionFloor(dir)).not.toThrow(); + }); + }); +}); + +describe("MAC_MIN_OS_FLOOR", () => { + it("matches the floor the .app declares to LaunchServices", () => { + // Shared parser rather than a regex of its own: electron-builder.json5 is heavily + // commented, its comments name this very key, and a private copy here is how the + // two guards drift into one hardened and one not (see scripts/macos-floor.mjs). + const declared = declaredAppVersionFrom( + readFileSync(path.join(path.dirname(BEFORE_PACK), "..", "electron-builder.json5"), "utf8"), + ); + expect( + declared, + 'no "minimumSystemVersion" in the mac block of electron-builder.json5 — without ' + + "it the .app inherits Electron's own floor, which is what let #515 ship", + ).not.toBeNull(); + + const { MAC_MIN_OS_FLOOR } = testing(); + const norm = (v) => v.split(".").concat(["0", "0"]).slice(0, 2).join("."); + // Equal, not merely <=: a pack-time guard looser than the app's own declaration + // would wave through exactly the binaries LaunchServices then refuses to run. + expect(norm(MAC_MIN_OS_FLOOR)).toBe(norm(declared)); + }); +}); + +// #616: a macOS .app that ships the libav dylibs but no ffmpeg CLI transcribes nothing on +// machines without a system ffmpeg — resolveFfmpeg() finds no candidate, native extraction +// throws, and the renderer can only show "Failed to fetch". The payload guard is the only +// thing that can catch that before the DMG exists; this pins it to the requirement. +describe("MAC_REQUIRED", () => { + it("demands the ffmpeg CLI, and names transcription as what breaks without it", () => { + const { MAC_REQUIRED } = testing(); + const entry = MAC_REQUIRED.find((req) => req.match("ffmpeg")); + expect(entry, "MAC_REQUIRED has no entry matching a file named 'ffmpeg'").toBeDefined(); + expect(entry.breaks).toContain("#616"); + }); + + it("refuses a payload that has everything except the ffmpeg binary", () => { + const { MAC_REQUIRED, checkNativePayload } = testing(); + // One satisfying file per requirement, spelled out rather than derived from the + // matchers (which cannot be inverted) — except the ffmpeg CLI, deliberately absent. + const files = Object.fromEntries( + [ + "compositor_view.node", + "whisper-stt-server", + "libggml-base.dylib", + "openscreen-screencapturekit-helper", + "libavdevice.62.dylib", + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map( + (lib, i) => `lib${lib}.${62 - i}.dylib`, + ), + ].map((name) => [name, Buffer.from("x")]), + ); + withPayload(files, (dir) => { + expect(() => + checkNativePayload({ + dir, + required: MAC_REQUIRED, + osLabel: "macOS", + bundleNoun: "the .app", + emptyDirFix: "unused", + }), + ).toThrow(/ffmpeg CLI/); + }); + }); + + it("refuses a payload whose ffmpeg CLI is missing libavdevice", () => { + const { MAC_REQUIRED, checkNativePayload } = testing(); + const files = Object.fromEntries( + [ + "compositor_view.node", + "ffmpeg", + "whisper-stt-server", + "libggml-base.dylib", + "openscreen-screencapturekit-helper", + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map( + (lib, i) => `lib${lib}.${62 - i}.dylib`, + ), + ].map((name) => [name, Buffer.from("x")]), + ); + withPayload(files, (dir) => { + expect(() => + checkNativePayload({ + dir, + required: MAC_REQUIRED, + osLabel: "macOS", + bundleNoun: "the .app", + emptyDirFix: "unused", + }), + ).toThrow(/libavdevice/); + }); + }); +}); diff --git a/scripts/build-linux-compositor-addon.mjs b/scripts/build-linux-compositor-addon.mjs index 9cf1b7352..717d1b79f 100644 --- a/scripts/build-linux-compositor-addon.mjs +++ b/scripts/build-linux-compositor-addon.mjs @@ -16,7 +16,7 @@ // (ensureFfmpegSharedDllsOnPath), but glibc reads LD_LIBRARY_PATH once at // process start, so the equivalent trick cannot work after Electron is // already running. Instead the addon is linked with `-rpath,$ORIGIN` and -// the five ffmpeg sonames are copied next to it, which makes the .node +// the six ffmpeg sonames are copied next to it, which makes the .node // self-contained wherever it is installed — no env var, no PATH surgery. import { spawnSync } from "node:child_process"; @@ -36,6 +36,7 @@ const FFMPEG_SONAMES = [ "libavutil.so.60", "libswscale.so.9", "libswresample.so.6", + "libavfilter.so.11", ]; const run = (command, args, options = {}) => diff --git a/scripts/build-linux-pipewire-helper.mjs b/scripts/build-linux-pipewire-helper.mjs index ac898c4c2..b6928c3a8 100644 --- a/scripts/build-linux-pipewire-helper.mjs +++ b/scripts/build-linux-pipewire-helper.mjs @@ -143,7 +143,7 @@ function stageFfmpeg(dir) { // Only the sonames the helper actually links, and only the real files — // the tree also holds unversioned `.so` symlinks that the loader never // consults at runtime. - const wanted = /^lib(avcodec|avformat|avutil|swscale|swresample)\.so\.\d+$/; + const wanted = /^lib(avcodec|avformat|avutil|avfilter|swscale|swresample)\.so\.\d+$/; let copied = 0; for (const entry of fs.readdirSync(source)) { if (!wanted.test(entry)) { diff --git a/scripts/build-macos-compositor-addon.mjs b/scripts/build-macos-compositor-addon.mjs index 200b9cd87..d1f52268c 100644 --- a/scripts/build-macos-compositor-addon.mjs +++ b/scripts/build-macos-compositor-addon.mjs @@ -130,6 +130,26 @@ function installAtomically(from, to) { fs.renameSync(tmp, to); } +/** + * Every Mach-O load command that points outside the OS's own prefixes. `otool -D`/`-L` + * both echo the filename on the first line, so both lists skip it. + */ +function absolutePaths(file) { + const id = execFileSync("otool", ["-D", file], { encoding: "utf8" }) + .split("\n") + .slice(1) // first line is the filename echoed back + .map((l) => l.trim()) + .filter(Boolean); + const deps = execFileSync("otool", ["-L", file], { encoding: "utf8" }) + .split("\n") + .slice(1) // ditto + .map((l) => l.trim().split(" ")[0]) + .filter(Boolean); + return [...id, ...deps].filter( + (p) => p.startsWith("/") && !p.startsWith("/usr/lib/") && !p.startsWith("/System/"), + ); +} + /** * Vendors the ffmpeg dylibs next to the addon and rewrites every install name to * `@rpath`, so the packaged app loads its own copies instead of a build-machine path. @@ -204,22 +224,6 @@ function vendorFfmpegDylibs(nodePath, ffmpegDir) { // the id above — nor any future non-ffmpeg dependency that arrives absolute. // Assert the real invariant instead: nothing outside the OS's own prefixes // may be referenced by absolute path. - const absolutePaths = (file) => { - const id = execFileSync("otool", ["-D", file], { encoding: "utf8" }) - .split("\n") - .slice(1) // first line is the filename echoed back - .map((l) => l.trim()) - .filter(Boolean); - const deps = execFileSync("otool", ["-L", file], { encoding: "utf8" }) - .split("\n") - .slice(1) // ditto - .map((l) => l.trim().split(" ")[0]) - .filter(Boolean); - return [...id, ...deps].filter( - (p) => p.startsWith("/") && !p.startsWith("/usr/lib/") && !p.startsWith("/System/"), - ); - }; - for (const file of [nodePath, ...names.map((n) => path.join(outDir, n))]) { const remaining = absolutePaths(file); if (remaining.length > 0) { @@ -233,6 +237,82 @@ function vendorFfmpegDylibs(nodePath, ffmpegDir) { console.log(`No absolute build-machine paths remain in ${path.basename(nodePath)} or its dylibs`); } +/** + * Stages the SDK's `ffmpeg` BINARY next to the addon's vendored dylibs, so the packaged + * app has the CLI that `resolveFfmpeg()` (electron/media/audioPeaks.ts) and native STT + * audio extraction (electron/stt/extractAudio.ts) spawn. + * + * Without it the .app carries libav*.dylib but no executable, and every transcription on + * a machine without a system ffmpeg dies with FfmpegUnavailableError — the failure the + * renderer can only show as "Failed to fetch" (#616). Windows never had this gap: its + * installer ships `ffmpeg-shared.exe` beside the DLLs it links. + * + * The binary leaves `make install` referencing the SDK's lib/ — by absolute path or by + * `@executable_path/../lib/`, depending on the configure — and neither survives the move + * into electron-builder's extraResources. Rewriting every lib reference to `@rpath/` + * and adding `@loader_path` points it at the very dylibs `vendorFfmpegDylibs` put beside + * it, whose ids already are `@rpath/`. + */ +function stageFfmpegBinary(outDir, ffmpegDir) { + const from = path.join(ffmpegDir, "bin", "ffmpeg"); + if (!fs.existsSync(from)) { + throw new Error(`No ffmpeg binary at ${from}; the SDK tree is incomplete.`); + } + const to = path.join(outDir, "ffmpeg"); + installAtomically(from, to); + fs.chmodSync(to, 0o755); + + const deps = execFileSync("otool", ["-L", to], { encoding: "utf8" }) + .split("\n") + .map((line) => line.trim().split(" ")[0]) + .filter((p) => /(^\/|@executable_path).*lib(av|sw)\w+\.\d+\.dylib$/.test(p)); + if (deps.length === 0) { + throw new Error(`${to} links no ffmpeg dylib — nothing to rewrite, which is wrong.`); + } + // The CLI links more of the SDK than the addon does (libavdevice, libpostproc, …). + // Refresh every direct dependency, including files vendorFfmpegDylibs already staged. + // The output directory persists between local builds, so keeping an existing file could + // mix a new CLI with an old SDK dylib or preserve a half-rewritten file from an interrupted + // run. Replacing the complete set also gives every file the same atomic-install guarantee + // as the addon and CLI. + const stagedLibraries = []; + for (const dep of new Set(deps)) { + const name = path.basename(dep); + const staged = path.join(outDir, name); + const sdkLib = path.join(ffmpegDir, "lib", name); + if (!fs.existsSync(sdkLib)) { + throw new Error(`Missing ${sdkLib}; the binary links it but the SDK does not ship it.`); + } + installAtomically(sdkLib, staged); + fs.chmodSync(staged, 0o755); + execFileSync("install_name_tool", ["-id", `@rpath/${name}`, staged]); + stagedLibraries.push(staged); + } + for (const file of [...stagedLibraries, to]) { + const fileDeps = execFileSync("otool", ["-L", file], { encoding: "utf8" }) + .split("\n") + .map((line) => line.trim().split(" ")[0]) + .filter((p) => /(^\/|@executable_path).*lib(av|sw)\w+\.\d+\.dylib$/.test(p)); + for (const dep of fileDeps) { + execFileSync("install_name_tool", ["-change", dep, `@rpath/${path.basename(dep)}`, file]); + } + execFileSync("install_name_tool", ["-add_rpath", "@loader_path", file]); + // install_name_tool invalidates the signature; re-sign ad-hoc. + execFileSync("codesign", ["--force", "--sign", "-", file]); + } + + for (const file of [...stagedLibraries, to]) { + const remaining = absolutePaths(file); + if (remaining.length > 0) { + throw new Error( + `${path.basename(file)} still references build-machine paths after rewriting: ` + + remaining.join(", "), + ); + } + } + console.log(`Staged ffmpeg binary at ${to} (rewritten to @rpath beside the vendored dylibs)`); +} + /** * Refuses to package a GPL ffmpeg. `--enable-gpl` pulls x264/x265 in and relicenses this * MIT app; a Homebrew ffmpeg is exactly that and is an easy thing to point MAC_FFMPEG_DIR @@ -270,6 +350,7 @@ installAtomically(builtDylib, archDest); // Only the arch-tagged copy ships (mac `extraResources`, filter `darwin-*/*`), so that // is the one that gets its dylibs and its @rpath. vendorFfmpegDylibs(archDest, macFfmpegDir); +stageFfmpegBinary(archBinDir, macFfmpegDir); console.log(`Built ${builtDylib}`); console.log(`Copied ${dest}`); diff --git a/scripts/build-whisper-stt.sh b/scripts/build-whisper-stt.sh index dc4728141..74f5be046 100644 --- a/scripts/build-whisper-stt.sh +++ b/scripts/build-whisper-stt.sh @@ -73,6 +73,11 @@ os_arch_tag() { readonly OS_ARCH="$(os_arch_tag)" readonly OUT_DIR="${OUT_ROOT}/${OS_ARCH}" +# Kept beside the other build constants so it is greppable next to the ffmpeg one in +# scripts/fetch-ffmpeg-macos.mjs; the two must agree, and both must match +# `mac.minimumSystemVersion` in electron-builder.json5. +readonly MACOS_DEPLOYMENT_TARGET="13.0" + # Determine the default backend flag for this host. backend_flag_for_host() { case "${OS_ARCH}" in @@ -305,6 +310,21 @@ BUILD_FLAGS=() if [[ -n "${DEFAULT_FLAG}" ]]; then BUILD_FLAGS+=("${DEFAULT_FLAG}") fi +# Pin the macOS floor the app actually ships against (`mac.minimumSystemVersion` in +# electron-builder.json5). Without it CMake +# defaults the deployment target to the BUILD MACHINE's SDK, so whisper-stt-server and +# the libwhisper/libggml*/libparakeet dylibs inherit whatever macOS built them — +# measured 26.0 on the shipped v1.10.0 payload, and ~15.x from CI's `macos-latest`, +# a floor that moves on its own whenever GitHub rolls that image. +# +# This is the macOS twin of the ubuntu-22.04 pin in build-whisper-stt.yml: same defect +# (a shipped binary's floor decided by the runner rather than by the project), different +# libc. Note it is NOT a loader version gate — dyld does not refuse a binary whose minos +# exceeds the running OS. Setting it is what makes the linker enforce macOS 12 symbol +# availability, which is what actually fails at load time. See issue #515. +if [[ "${OS_ARCH}" == darwin-* ]]; then + BUILD_FLAGS+=("-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_DEPLOYMENT_TARGET}") +fi # See the comment in build_variant() re: bash 3.2 + `set -u` + empty arrays # (macOS x64/CPU has no DEFAULT_FLAG, so BUILD_FLAGS is genuinely empty here). build_variant "default" ${BUILD_FLAGS[@]+"${BUILD_FLAGS[@]}"} diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs index 063d81b30..8d4cfc018 100644 --- a/scripts/build-windows-wgc-helper.mjs +++ b/scripts/build-windows-wgc-helper.mjs @@ -95,3 +95,13 @@ console.log(`Built ${outputPath}`); console.log(`Copied ${distributablePath}`); console.log(`Built ${cursorSamplerOutputPath}`); console.log(`Copied ${cursorSamplerDistributablePath}`); + +const audioUtilsTestPath = path.join(BUILD_DIR, "audio_sample_utils_test.exe"); +if (!fs.existsSync(audioUtilsTestPath)) { + throw new Error(`WGC helper build completed but ${audioUtilsTestPath} was not found.`); +} +// Snap/resample unit tests must pass. Media Foundation AAC probes skip on +// hosts without the stock encoder (Windows N/KN, Server without Media Feature +// Pack) instead of failing this packaging command. +await run(audioUtilsTestPath, [], { cwd: BUILD_DIR }); +console.log(`Passed ${audioUtilsTestPath}`); diff --git a/scripts/check-docs.mjs b/scripts/check-docs.mjs index 65119765e..f1c671962 100644 --- a/scripts/check-docs.mjs +++ b/scripts/check-docs.mjs @@ -27,6 +27,9 @@ const LEGACY = [ "Titlebar", "TranscriptEditor", "BackgroundPane", + "LeftRail", + "MediaPane", + "SourceTranscriptModal", "ai-edition-roadmap", "ai-edition-collision-analysis", "openscreen-inventory", diff --git a/scripts/check-macos-deployment-target.test.mjs b/scripts/check-macos-deployment-target.test.mjs new file mode 100644 index 000000000..d405fa241 --- /dev/null +++ b/scripts/check-macos-deployment-target.test.mjs @@ -0,0 +1,146 @@ +// Guards the macOS deployment floor of the native Swift helpers (issue #515). +// +// The floor is declared in THREE places that must agree: `mac.minimumSystemVersion` in +// electron-builder.json5 (what the .app tells LaunchServices), the README's system +// requirements (what we promise), and the `platforms:` block in Package.swift (what the +// helpers are actually built for). This file ties the third to the first. +// +// The direction matters. Package.swift may not declare a floor HIGHER than the app +// advertises — that is exactly #515: the floor here was set to 13 when ScreenCaptureKit +// was the only target, openscreen-macos-cursor-helper was added later and inherited it +// because SwiftPM has no per-target override, and the bundle went on advertising macOS 12 +// (Electron's own LSMinimumSystemVersion, inherited because the key was unset). +// +// The damage was not the version number. At a deployment target >= 13 the linker resolves +// the Swift Foundation overlay symbols against Foundation.framework and drops +// /usr/lib/swift/libswiftFoundation.dylib from the load commands; on macOS 12 those +// symbols live only in that dylib, so the helper died in the loader before it could speak +// — and the app reported that as a denied Accessibility grant. +// +// A text assertion rather than a build: this has to fail on Linux and Windows CI too, +// where no Swift toolchain exists. + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +import { declaredAppFloorFrom } from "./macos-floor.mjs"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const PACKAGE_SWIFT = path.join(ROOT, "electron", "native", "screencapturekit", "Package.swift"); +const BUILDER_CONFIG = path.join(ROOT, "electron-builder.json5"); + +function declaredAppFloor() { + return declaredAppFloorFrom(readFileSync(BUILDER_CONFIG, "utf8")); +} + +/** + * Reads the major version out of the `platforms:` block, accepting both spellings + * SwiftPM allows — `.macOS(.v12)` and `.macOS("12.3")`. + */ +function declaredMacOsFloor(source) { + // Scoped to the platforms block, with comments stripped from it, rather than matched + // across the whole manifest. That block is preceded by a long comment discussing these + // very version numbers, so a file-wide match is one careless edit away from reading the + // prose instead of the declaration — and reporting a floor the build does not use is + // the one failure this guard must not have. + const block = source.match(/\bplatforms\s*:\s*\[([\s\S]*?)\]/)?.[1]; + if (!block) { + return null; + } + const declarations = block.replace(/\/\/[^\n]*/g, ""); + + const enumMatch = declarations.match(/\.macOS\(\s*\.v(\d+)(?:_\d+)?\s*\)/); + if (enumMatch) { + return Number(enumMatch[1]); + } + + const stringMatch = declarations.match(/\.macOS\(\s*"(\d+)(?:\.\d+)*"\s*\)/); + return stringMatch ? Number(stringMatch[1]) : null; +} + +describe("macOS native helper deployment target", () => { + const source = readFileSync(PACKAGE_SWIFT, "utf8"); + + it("declares a floor no higher than the app itself advertises", () => { + const floor = declaredMacOsFloor(source); + const appFloor = declaredAppFloor(); + + expect(floor, `no .macOS(...) platform found in ${PACKAGE_SWIFT}`).not.toBeNull(); + expect( + appFloor, + 'no "minimumSystemVersion" found in electron-builder.json5 — without it the .app ' + + "inherits Electron's own floor, which is what let #515 ship", + ).not.toBeNull(); + expect( + floor, + `Package.swift builds the native helpers for macOS ${floor}, above the ${appFloor} ` + + "the .app advertises to LaunchServices. This block is package-wide and also " + + "governs openscreen-macos-cursor-helper, which needs nothing newer than 10.15. " + + "Every user between the two versions gets a helper that dies in the loader, " + + "reported as a denied Accessibility grant. See issue #515.", + ).toBeLessThanOrEqual(appFloor); + }); + + it("parses both spellings SwiftPM accepts", () => { + expect(declaredMacOsFloor("platforms: [ .macOS(.v12) ]")).toBe(12); + expect(declaredMacOsFloor("platforms: [ .macOS(.v10_15) ]")).toBe(10); + expect(declaredMacOsFloor('platforms: [ .macOS("12.3") ]')).toBe(12); + expect(declaredMacOsFloor("platforms: [ .iOS(.v16) ]")).toBeNull(); + }); + + it("reads the mac block's floor, not the first match in the file", () => { + // Both failure shapes the real config invites: a comment discussing the key + // (electron-builder.json5 carries a long one directly above it), and another + // platform block that could grow the same key later. + const decoyComment = [ + '// was "minimumSystemVersion": "12.0" before #515', + '"mac": {', + '\t"minimumSystemVersion": "13.0",', + "}", + ].join("\n"); + expect(declaredAppFloorFrom(decoyComment)).toBe(13); + + const decoySibling = [ + '"win": {', + '\t"minimumSystemVersion": "99.0",', + "},", + '"mac": {', + '\t"minimumSystemVersion": "13.0",', + "}", + ].join("\n"); + expect(declaredAppFloorFrom(decoySibling)).toBe(13); + + // A URL's `//` must survive the comment strip, or the mac block is lost with it. + const withUrl = [ + '"publish": [{ "url": "https://example.invalid/feed" }],', + '"mac": {', + '\t"minimumSystemVersion": "13.0",', + "}", + ].join("\n"); + expect(declaredAppFloorFrom(withUrl)).toBe(13); + + expect(declaredAppFloorFrom('"win": { "minimumSystemVersion": "13.0" }')).toBeNull(); + }); + + it("reads the declaration, not prose that happens to mention a version", () => { + // The real manifest carries exactly this shape: a comment about the floor sitting + // directly above the floor. Matching file-wide would report 12 while the build used + // 13 — a guard that passes for the very bug it exists to catch. + const decoyAbove = [ + "// It was .macOS(.v12) until this changed; see issue #515.", + "platforms: [", + "\t.macOS(.v13)", + "],", + ].join("\n"); + expect(declaredMacOsFloor(decoyAbove)).toBe(13); + + const decoyInside = ["platforms: [", "\t// was .macOS(.v12)", "\t.macOS(.v13)", "],"].join( + "\n", + ); + expect(declaredMacOsFloor(decoyInside)).toBe(13); + + expect(declaredMacOsFloor("// .macOS(.v12) with no platforms block at all")).toBeNull(); + }); +}); diff --git a/scripts/convert-selfie-segmentation-to-onnx.py b/scripts/convert-selfie-segmentation-to-onnx.py new file mode 100644 index 000000000..3cb617af2 --- /dev/null +++ b/scripts/convert-selfie-segmentation-to-onnx.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Convert the vendored MediaPipe SelfieSegmentation .tflite to ONNX. + +This is the script that produced `selfie_segmentation_landscape.onnx`. It is checked in so +the artifact is reproducible and auditable rather than an opaque binary: the conversion is +NOT a mechanical one-liner, it needs two hand repairs (below), and anyone reviewing the model +needs to be able to see and re-run them. + +It is not part of any build. Nothing in the app runs Python; this is provenance tooling, run +by hand on the rare occasion the model is regenerated. + + pip install "numpy<2" "tensorflow==2.13.1" "tf2onnx==1.16.1" "onnx==1.16.2" "protobuf<4" + python scripts/convert-selfie-segmentation-to-onnx.py landscape + +Why the repairs are needed +-------------------------- +`tf2onnx` reports success on this model but leaves 12 operators that ONNX Runtime cannot load: + +1. **11 x HardSwish emitted into an opset-13 graph.** `HardSwish` is opset 14+, so the graph + is invalid as declared. Fixed by raising the opset to 16. + +2. **1 x TFL_Convolution2DTransposeBias** — a MediaPipe *custom* operator with no ONNX + equivalent, so tf2onnx passes it through under the default domain where it does not exist. + It is the last convolution before the output sigmoid: a 2x2 stride-2 transposed + convolution, 16 channels in, 1 out, plus a bias. Rewritten here as a native + `ConvTranspose` + bias, with the weights transposed from TFLite's + `[C_out, kH, kW, C_in]` to ONNX's `[C_in, C_out/group, kH, kW]`. + + tf2onnx also inserts an NHWC `Transpose` to feed that custom node. Since the replacement + consumes NCHW directly, the transpose is dropped and the layout flip moves after the + sigmoid (sigmoid is elementwise, so the order is equivalent). + +The result passes `onnx.checker.check_model(..., full_check=True)` and, on a real webcam +frame, produces a mask identical between the CPU and DirectML execution providers. + +The graph is fully convolutional and resolution-agnostic (every `Reshape` target is +channel-only, every `Resize` uses scales rather than sizes), so the input dimensions can be +rewritten after the fact -- but **both dimensions must be divisible by 16** or the +skip-connection `Add`s fail on mismatched extents. + +Source model: `public/mediapipe/selfie_segmentation/*.tflite`, vendored from MediaPipe +(Apache-2.0). This conversion is a derived work of that file; no weights are downloaded. +""" +import argparse +import pathlib +import subprocess +import sys + +import numpy as np +import onnx +from onnx import helper, numpy_helper, shape_inference + +HERE = pathlib.Path(__file__).resolve().parent +MODELS = HERE.parent / "public" / "mediapipe" / "selfie_segmentation" + +VARIANTS = { + "landscape": ("selfie_segmentation_landscape.tflite", "selfie_segmentation_landscape.onnx"), + "square": ("selfie_segmentation.tflite", "selfie_segmentation.onnx"), +} + + +def repair(src: pathlib.Path, dst: pathlib.Path) -> None: + model = onnx.load(str(src)) + graph = model.graph + + for opset in model.opset_import: + if opset.domain in ("", "ai.onnx"): + opset.version = 16 # HardSwish is opset 14+ + + init = {i.name: numpy_helper.to_array(i) for i in graph.initializer} + custom = next(n for n in graph.node if n.op_type == "TFL_Convolution2DTransposeBias") + feed_transpose = next(n for n in graph.node if n.output[0] == custom.input[0]) + source = feed_transpose.input[0] # NCHW feature map + sigmoid = next(n for n in graph.node if custom.output[0] in n.input) + assert sigmoid.op_type == "Sigmoid", sigmoid.op_type + + weights = init[custom.input[1]] # [C_out, kH, kW, C_in] + assert weights.shape == (1, 2, 2, 16), weights.shape + graph.initializer.append( + numpy_helper.from_array(np.transpose(weights, (3, 0, 1, 2)).copy(), "convT_W") + ) + + out_name = graph.output[0].name + nodes = [n for n in graph.node if n not in (feed_transpose, custom, sigmoid)] + nodes += [ + helper.make_node( + "ConvTranspose", [source, "convT_W", custom.input[2]], ["convT_out"], + name="conv2d_transpose_native", + kernel_shape=[2, 2], strides=[2, 2], pads=[0, 0, 0, 0], + ), + helper.make_node("Sigmoid", ["convT_out"], ["mask_nchw"], name="segment_sigmoid"), + helper.make_node("Transpose", ["mask_nchw"], [out_name], name="mask_to_nhwc", + perm=[0, 2, 3, 1]), + ] + del graph.node[:] + graph.node.extend(nodes) + del graph.value_info[:] + + model = shape_inference.infer_shapes(model, strict_mode=True) + onnx.checker.check_model(model, full_check=True) + onnx.save(model, str(dst)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("variant", choices=sorted(VARIANTS), nargs="?", default="landscape") + args = parser.parse_args() + + tflite_name, onnx_name = VARIANTS[args.variant] + tflite = MODELS / tflite_name + if not tflite.exists(): + print(f"missing source model: {tflite}", file=sys.stderr) + return 1 + + raw = MODELS / f".{onnx_name}.raw" + subprocess.run( + [sys.executable, "-m", "tf2onnx.convert", "--tflite", str(tflite), + "--output", str(raw), "--opset", "13"], + check=True, + ) + # tf2onnx exits 0 while leaving 12 unloadable operators behind -- see the module docstring. + repair(raw, MODELS / onnx_name) + raw.unlink(missing_ok=True) + + model = onnx.load(str(MODELS / onnx_name)) + gi, go = model.graph.input[0], model.graph.output[0] + shape = lambda v: [d.dim_value for d in v.type.tensor_type.shape.dim] + leftover = [n.op_type for n in model.graph.node if n.domain not in ("", "ai.onnx")] + print(f"wrote {MODELS / onnx_name}") + print(f" input {gi.name} {shape(gi)}") + print(f" output {go.name} {shape(go)}") + print(f" nodes {len(model.graph.node)} non-standard ops: {leftover or 'none'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fetch-ffmpeg-macos.mjs b/scripts/fetch-ffmpeg-macos.mjs index d611aa627..d165af762 100644 --- a/scripts/fetch-ffmpeg-macos.mjs +++ b/scripts/fetch-ffmpeg-macos.mjs @@ -29,6 +29,20 @@ const ROOT = path.join(__dirname, ".."); const CRATES_DIR = path.join(ROOT, "crates"); /** Pinned release. The directory name is what build.rs looks for. */ +// The macOS floor the app ships against — keep in step with `mac.minimumSystemVersion` +// in electron-builder.json5, which is what the .app tells LaunchServices. +// +// Without it, clang defaults the deployment target +// to the BUILD MACHINE's SDK, so the vendored dylibs inherit whatever macOS built them — +// measured 26.0 on a local build and ~15.x from CI's `macos-latest`, a floor that moves on +// its own every time GitHub rolls that image. That is the same class of leak the configure +// comment below guards against for Homebrew packages, and it is the one it missed. +// +// Note this is NOT a loader version gate: dyld does not refuse a dylib whose minos exceeds +// the running OS (verified). Setting it is what makes the LINKER enforce macOS 12 symbol +// availability, which is the thing that actually breaks at load time. See issue #515. +const MACOS_DEPLOYMENT_TARGET = "13.0"; + const VERSION = "8.1.2"; const TARBALL_SHA256 = "464beb5e7bf0c311e68b45ae2f04e9cc2af88851abb4082231742a74d97b524c"; const DEST = path.join(CRATES_DIR, "thirdparty", `ffmpeg-n${VERSION}-macos64-lgpl-shared`); @@ -148,22 +162,57 @@ function isLgpl(dir) { return /Lesser General Public/i.test(banner) && !/GNU General Public License/i.test(banner); } +/** + * Whether the vendored tree was built for the pinned deployment target. Part of the + * reuse decision alongside the licence: a tree that predates the pin (or was built by + * hand without it) is LGPL and would otherwise be reused forever, only for + * before-pack's floor guard to refuse it at packaging time — a five-minute rebuild + * deferred to the worst possible moment. The binary, not the dylibs, because it is one + * vtool call and the whole tree shares a configure. + */ +function isAtDeploymentTarget(dir) { + const bin = path.join(dir, "bin", "ffmpeg"); + if (!fs.existsSync(bin)) return false; + const build = execFileSync("vtool", ["-show-build", bin], { encoding: "utf8" }); + const minos = /minos (\d+(?:\.\d+)+)/.exec(build)?.[1]; + if (minos === undefined) return false; + const compareVersions = (a, b) => { + const left = a.split(".").map(Number); + const right = b.split(".").map(Number); + for (let i = 0; i < Math.max(left.length, right.length); i++) { + if ((left[i] ?? 0) !== (right[i] ?? 0)) return (left[i] ?? 0) - (right[i] ?? 0); + } + return 0; + }; + return compareVersions(minos, MACOS_DEPLOYMENT_TARGET) <= 0; +} + if (process.platform !== "darwin") { console.log("Skipping macOS ffmpeg vendoring: macOS-only (Windows uses fetch:ffmpeg)."); process.exit(0); } -if (fs.existsSync(path.join(DEST, "include")) && isLgpl(DEST)) { - console.log(`ffmpeg already vendored at ${DEST} and its -L banner says LGPL. Nothing to do.`); +if (fs.existsSync(path.join(DEST, "include")) && isLgpl(DEST) && isAtDeploymentTarget(DEST)) { + console.log( + `ffmpeg already vendored at ${DEST}: LGPL, built for macOS ${MACOS_DEPLOYMENT_TARGET}. Nothing to do.`, + ); process.exit(0); } -if (fs.existsSync(path.join(DEST, "include"))) { +if (fs.existsSync(path.join(DEST, "include")) && !isLgpl(DEST)) { throw new Error( `${DEST} exists but is not an LGPL build (checked with \`ffmpeg -L\`).\n` + "Refusing to reuse it — linking a GPL ffmpeg would relicense OpenScreen.\n" + "Delete the directory and re-run to rebuild it from source.", ); } +if (fs.existsSync(path.join(DEST, "include"))) { + console.warn( + `${DEST} was not built for the ${MACOS_DEPLOYMENT_TARGET} deployment target ` + + "(check: vtool -show-build). Rebuilding — a stale floor here only fails at packaging,\n" + + "in before-pack's macOS version guard.", + ); + fs.rmSync(DEST, { recursive: true, force: true }); +} const work = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-ffmpeg-")); const tarball = path.join(work, `ffmpeg-${VERSION}.tar.xz`); @@ -202,6 +251,10 @@ run( "--disable-x86asm", `--arch=${process.arch === "arm64" ? "arm64" : "x86_64"}`, "--cc=clang", + // Both, not just cflags: the deployment target has to reach the link step too, or + // the dylibs are stamped with the build machine's floor however they were compiled. + `--extra-cflags=-mmacosx-version-min=${MACOS_DEPLOYMENT_TARGET}`, + `--extra-ldflags=-mmacosx-version-min=${MACOS_DEPLOYMENT_TARGET}`, ], { cwd: src }, ); diff --git a/scripts/fetch-ffmpeg.mjs b/scripts/fetch-ffmpeg.mjs index 8c2f0ed83..222485d97 100644 --- a/scripts/fetch-ffmpeg.mjs +++ b/scripts/fetch-ffmpeg.mjs @@ -156,8 +156,23 @@ const GPL_LIBS = [ /** Worse than GPL: these make the binary unredistributable at all. */ const NONFREE_LIBS = ["libfdk-aac", "libfdk_aac"]; -/** The encoders the export path actually selects, per platform. */ -const WANTED_ENCODERS = { +/** + * The hardware encoders we look for in a vendored build, per platform. + * + * REPORTED, NOT VERIFIED — and the distinction has bitten. Presence in + * `-encoders` means the build was compiled with the wrapper; it says nothing + * about whether the encoder RUNS on a given machine. The vendored builds list + * `h264_vaapi` on Linux, yet on any host with libva < 2.21 (Ubuntu 24.04 LTS + * included) it used to take the whole process down with SIGABRT rather than + * return an error, because the implib trampoline `abort()`s on a symbol it + * cannot resolve. See issues #552 and #576. + * + * So this list drives a log line and nothing else. The only evidence that an + * encoder works is a frame going through it — `vaapi_encodes_from_an_exported_dmabuf` + * in `crates/compositor/src/pipeline_linux.rs` is that evidence for the Linux + * hardware path, and the export falls back to software whenever it is absent. + */ +const REPORTED_ENCODERS = { win32: ["h264_nvenc", "h264_qsv", "h264_amf"], linux: ["h264_nvenc", "h264_vaapi"], }; @@ -230,13 +245,14 @@ function assertLgpl(exePath, extraEnv) { function reportEncoders(exePath, platform) { const encoders = run(exePath, ["-hide_banner", "-encoders"]).stdout ?? ""; - const wanted = WANTED_ENCODERS[platform] ?? []; + const wanted = REPORTED_ENCODERS[platform] ?? []; const found = wanted.filter((e) => new RegExp(`\\s${e}\\s`).test(encoders)); const missing = wanted.filter((e) => !found.includes(e)); - console.log(` hardware encoders: ${found.join(", ") || "(none)"}`); + console.log(` hardware encoders present (not verified): ${found.join(", ") || "(none)"}`); if (missing.length > 0) { - // Not fatal: which encoders a build exposes is separate from which GPU the - // machine has. selectVideoEncoder() probes at runtime regardless. + // Not fatal, and deliberately so: which encoders a build EXPOSES is + // separate both from which GPU the machine has and from whether the + // encoder is usable there at all. The runtime probes regardless. console.log(` not in this build: ${missing.join(", ")}`); } } @@ -407,13 +423,33 @@ async function fetchSharedDlls(tag, binDir) { return; } - // probe for any previously vendored DLL by name; re-download is driven by - // --force same as the static exe, checked once we know what we'd extract. - const alreadyVendored = - process.platform === "win32" && + // Completeness probe, not a mere existence probe. The compositor addon now + // links six shared ffmpeg DLLs — see crates/compositor/build.rs + // (avcodec, avformat, avutil, swresample, swscale, avfilter). A warm dev/CI + // tree that already holds the five pre-avfilter DLLs would satisfy an "any + // av*.dll is present" check and let `avfilter-11.dll` go un-vendored, breaking + // require() at runtime (OpenScreen#371 review, EtienneLescot). Require all + // six explicitly so a missing one forces a re-vendor. + const REQUIRED_SHARED_DLLS = [ + "avcodec", + "avformat", + "avutil", + "swresample", + "swscale", + "avfilter", + ]; + fs.mkdirSync(binDir, { recursive: true }); + const vendoredFiles = new Set( fs .readdirSync(binDir, { withFileTypes: true }) - .some((e) => e.isFile() && isSharedLib(e.name) && /^(lib)?av/i.test(e.name)); + .filter((e) => e.isFile()) + .map((e) => e.name), + ); + const alreadyVendored = + process.platform === "win32" && + REQUIRED_SHARED_DLLS.every((lib) => + [...vendoredFiles].some((f) => new RegExp(`^${lib}-\\d+\\.dll$`).test(f)), + ); // The build-time SDK comes out of this same archive, so a tree that has the // DLLs but not the SDK must still re-download — otherwise we skip here and // the compositor build fails afterwards on the missing FFMPEG_DIR. diff --git a/scripts/fetch-onnxruntime.mjs b/scripts/fetch-onnxruntime.mjs new file mode 100644 index 000000000..b97af4d40 --- /dev/null +++ b/scripts/fetch-onnxruntime.mjs @@ -0,0 +1,364 @@ +// Provisions the ONNX Runtime shared library into +// electron/native/bin/-/, next to the compositor addon and the +// ffmpeg libraries. That directory is gitignored and shipped by electron-builder's +// extraResources, so this runs at build time rather than committing a 15-38 MB binary. +// +// WHY IT SHIPS: the native compositor segments the webcam subject on the CPU +// execution provider (crates/compositor/src/segmentation.rs), and `ort` is linked +// with `load-dynamic` — nothing is needed to BUILD, but at runtime +// `ensureOnnxRuntimeOnPath` (electron/native-bridge/services/compositorViewService.ts) +// walks this exact directory looking for the library and sets ORT_DYLIB_PATH to it. +// Without it `Segmenter::load` fails, the compositor logs one line and draws the +// webcam unsegmented — so the AI background cutout/blur/custom modes are simply off. +// Everything degrades; nothing breaks. That is why this script never fails a build. +// +// VERSION IS NOT FREE TO MOVE. crates/Cargo.toml pins `ort` with feature `api-NN`, +// which is the MINIMUM ONNX Runtime minor version the crate will accept — a lower +// one makes `GetApi` return null and `ort` panics rather than erroring. The pin here +// must satisfy that, and scripts/fetch-onnxruntime.test.mjs cross-checks the two so +// a bump on either side cannot land alone. +// +// SUPPLY CHAIN. This binary is signed and shipped to every user, so nothing floats: +// - Pinned to an immutable release tag, never `latest`. +// - SHA-256 verified before the archive is opened. The digests below are the ones +// GitHub publishes per asset (`digest` in the releases API), independently +// re-verified by downloading and hashing. +// - Only the plain CPU assets. The `gpu_cuda*` variants are 200-320 MB and pull +// NVIDIA runtime dependencies we neither need nor may redistribute; the measured +// decision to use the CPU EP is in +// technical-documentation/engineering/webcam-segmentation.md. +// +// LICENSING: ONNX Runtime is MIT, which is compatible with this MIT app — but the +// archive is checked rather than trusted, the same way fetch-ffmpeg.mjs verifies +// ffmpeg's LGPL-ness instead of believing the asset name. Attribution ships in +// THIRD-PARTY-NOTICES.md. + +import { spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); + +/** + * The pinned release. Asset names are DERIVED from it rather than written out per + * entry, which is deliberate: fetch-ffmpeg.mjs keeps full asset strings and grew a + * test because a re-pin moved some and not others. Templating removes that failure + * mode by construction. The digests still have to move by hand — but a stale one + * fails loudly on the SHA-256 check before anything is extracted, which is the safe + * direction to fail in. + */ +const VERSION = "1.27.1"; + +/** + * The upstream commit `v${VERSION}` pointed at when it was reviewed. + * + * `v1.27.1` is a LIGHTWEIGHT tag — it points straight at a commit and can be moved by + * anyone with push rights upstream. Nothing here fetches source at install time, so this + * does not affect `npm run fetch:onnxruntime`; it matters to + * `.github/workflows/build-onnxruntime-macos.yml`, which builds the library and would + * otherwise attest an artifact to "whatever that tag meant that morning". The workflow + * resolves the tag and refuses to build if it no longer resolves here. + */ +const SOURCE_COMMIT = "df2ba1cf8108aa63627cf4cdf8f807880b938616"; +const BASE = `https://github.com/microsoft/onnxruntime/releases/download/v${VERSION}`; + +/** + * Where a target's archive is fetched from. Upstream unless the entry overrides it. + * + * The override exists for one reason, and it is not preference: **no published ONNX + * Runtime has ever satisfied this app's macOS floor.** Every release from 1.24 on is + * built for macOS 14, and every release before it for at least 13.3, while + * `electron-builder.json5` declares 13.0 and `before-pack.cjs` refuses anything above + * the floor — correctly, since the deployment target decides which symbols the linker + * resolves against the OS instead of emitting locally (#515). So `npm run build:mac` + * cannot package a macOS bundle at all with the upstream artifact. + * + * `.github/workflows/build-onnxruntime-macos.yml` builds one with the floor pinned and + * prints the `PINNED` entry to paste here. Everything else about this file is + * unchanged: an immutable URL, and a SHA-256 verified before the archive is opened. + * What moves is who built the bytes, not how much they are trusted. + */ +const baseUrlFor = (spec) => spec.baseUrl ?? BASE; + +/** + * Per-target: the upstream artifact slug, its digest, and the library to lift out. + * + * `out` is not cosmetic — it is the exact name `ortLibName()` looks for in + * compositorViewService.ts. `member` is the file inside the archive, which on macOS + * and Linux is the VERSIONED real file rather than the unversioned symlink beside + * it: tar restores that symlink as a symlink, and a dangling one in the packaged app + * would resolve to nothing. + * + * darwin-x64 is absent and cannot be added: Microsoft publishes no `osx-x86_64` + * (or universal) asset for any release from 1.27 on — arm64 is the only macOS + * target. Building it from source is the only way to change that, and it is not + * worth an ffmpeg-macos-sized build script for a shrinking platform when the + * fallback is "the effect is off". See the darwin-x64 branch in main(). + */ +const PINNED = { + "win32-x64": { + slug: "win-x64", + ext: "zip", + sha256: "2e00414a63fdef0914cd5a5ede6c707844878e0c08e1b6693842f0451b2df2a1", + member: "onnxruntime.dll", + out: "onnxruntime.dll", + }, + "win32-arm64": { + slug: "win-arm64", + ext: "zip", + sha256: "6e22c2061ba6400b42a59663d700c8694e4e8fe654cf452c4700c24237407ae1", + member: "onnxruntime.dll", + out: "onnxruntime.dll", + }, + // The one target that does NOT come from upstream, and it is not a preference. + // Microsoft's `onnxruntime-osx-arm64-1.27.1.tgz` is built for macOS 14, this app + // declares a 13.0 floor, and `before-pack.cjs` refuses a payload demanding more — + // correctly, since the deployment target decides which symbols the linker resolves + // against the OS rather than emitting locally (#515). So `npm run build:mac` could + // not package at all. No published release fixes that: every one from 1.24 on is + // `minos 14.0`, and every one before it is at least 13.3. + // + // This one is built by `.github/workflows/build-onnxruntime-macos.yml` from the + // commit above, with the deployment target pinned, and published under a + // `v0.0.0-*` tag — the marker this repo already uses for a binary that needs a + // permanent URL but is not a product version. Its provenance is attested: + // + // gh attestation verify onnxruntime-osx-arm64-1.27.1.tgz --repo getopenscreen/openscreen + // + // Everything else about it is unchanged: immutable URL, SHA-256 verified before + // the archive is opened. What moved is who built the bytes, not how far they are + // trusted. + "darwin-arm64": { + slug: "osx-arm64", + ext: "tgz", + sha256: "b8b7e62786eea42fc867bdbc2d3f573655a4b0e602c6938c4cea151a9ef71623", + member: `libonnxruntime.${VERSION}.dylib`, + out: "libonnxruntime.dylib", + baseUrl: + "https://github.com/getopenscreen/openscreen/releases/download/v0.0.0-onnxruntime-1.27.1", + }, + // Linux is wired into `build:linux` since its back-end gained the capture half + // (`capture_webcam_rgb` + `set_webcam_mask` in `compositor_linux.rs`). Until + // then this entry existed but was deliberately unused: the back-end carried the + // segmentation SHADER only, so `fx.z` never left 0 and the library would have + // been 23 MB of installer for a code path that could not run. + // See technical-documentation/engineering/webcam-segmentation-backend-port.md. + "linux-x64": { + slug: "linux-x64", + ext: "tgz", + sha256: "25b1ef1fea1acd210d63f8f24dc870ad6e077795ce1f54876252c6d3803c15af", + member: `libonnxruntime.so.${VERSION}`, + out: "libonnxruntime.so", + }, + "linux-arm64": { + slug: "linux-aarch64", + ext: "tgz", + sha256: "33c67e33d1e25b816878366ea276589a024f71f000e7ff955c4b33224d639edd", + member: `libonnxruntime.so.${VERSION}`, + out: "libonnxruntime.so", + }, +}; + +const assetName = (spec) => `onnxruntime-${spec.slug}-${VERSION}.${spec.ext}`; + +/** Magic bytes the vendored library must start with, per target platform. */ +const MAGIC = { + win32: { bytes: [0x4d, 0x5a], name: "PE (MZ)" }, // .dll + darwin: { bytes: [0xcf, 0xfa, 0xed, 0xfe], name: "Mach-O 64" }, // .dylib + linux: { bytes: [0x7f, 0x45, 0x4c, 0x46], name: "ELF" }, // .so +}; + +function run(cmd, args, opts = {}) { + return spawnSync(cmd, args, { stdio: "inherit", ...opts }); +} + +function tarBin() { + if (process.platform !== "win32") return "tar"; + const sys32 = path.join(process.env.SystemRoot ?? "C:\\Windows", "System32", "tar.exe"); + return fs.existsSync(sys32) ? sys32 : "tar"; +} + +function extract(archive, destDir) { + fs.mkdirSync(destDir, { recursive: true }); + // Run from destDir with a bare filename: given an absolute Windows path, tar + // reads "C:\..." as host:path and tries to resolve a host called C. + const r = run(tarBin(), [archive.endsWith(".zip") ? "-xf" : "-xzf", path.basename(archive)], { + cwd: destDir, + }); + if (r.status !== 0) throw new Error(`tar failed to extract ${path.basename(archive)}`); +} + +/** Depth-first search for a file by exact basename. */ +function find(dir, name) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) { + const hit = find(p, name); + if (hit) return hit; + } else if (entry.name === name) { + return p; + } + } + return null; +} + +/** + * Refuses anything that is not the MIT ONNX Runtime we pinned. + * + * Three independent checks, because the digest alone only proves we got the archive + * we asked for — it says nothing about having lifted the RIGHT FILE out of it, which + * is where a re-pin actually goes wrong (a renamed member silently vendors a 20 KB + * provider stub, and the failure surfaces as "the effect does nothing" months later). + * + * 1. the archive's LICENSE really is MIT — asset names are not evidence; + * 2. the library is a binary of the expected format for the target platform; + * 3. it carries the pinned version string, which is what `GetVersionString()` + * returns and what `ort` compares against its `api-NN` floor. + */ +function verify(libPath, licensePath, targetPlatform) { + const license = fs.readFileSync(licensePath, "utf8"); + if (!/^MIT License/m.test(license)) { + throw new Error( + `${path.basename(licensePath)} does not begin with "MIT License".\n` + + "Refusing to vendor: ONNX Runtime is MIT and this app is MIT — a relicensed\n" + + "upstream is a decision for a human, not a build script.", + ); + } + + const buf = fs.readFileSync(libPath); + const magic = MAGIC[targetPlatform]; + if (!magic.bytes.every((b, i) => buf[i] === b)) { + const got = [...buf.subarray(0, 4)].map((b) => b.toString(16).padStart(2, "0")).join(" "); + throw new Error( + `${path.basename(libPath)} is not a ${magic.name} binary (starts with ${got}).\n` + + "The archive layout probably changed under the pin — check `member`.", + ); + } + + // The version lives in the binary as a plain NUL-terminated string. + if (!buf.includes(Buffer.from(`\0${VERSION}\0`, "latin1"))) { + throw new Error( + `${path.basename(libPath)} does not carry the version string ${VERSION}.\n` + + "Either the pin and the digest disagree, or the wrong member was extracted.", + ); + } + + return `MIT ONNX Runtime ${VERSION}, ${magic.name}, ${(buf.length / 1048576).toFixed(1)} MB`; +} + +async function download(spec) { + const asset = assetName(spec); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-ort-")); + console.log(`Downloading ${asset}\n from v${VERSION}`); + const res = await fetch(`${baseUrlFor(spec)}/${asset}`); + if (!res.ok) throw new Error(`Download failed: ${res.status} ${res.statusText}`); + const bytes = Buffer.from(await res.arrayBuffer()); + + // Before opening it: is this the exact artifact we pinned? + const got = crypto.createHash("sha256").update(bytes).digest("hex"); + if (got !== spec.sha256) { + fs.rmSync(tmp, { recursive: true, force: true }); + throw new Error( + `SHA-256 mismatch for ${asset}\n expected ${spec.sha256}\n got ${got}\n` + + "Refusing to extract. Either the pin is stale or the artifact changed under it.", + ); + } + console.log(` sha256 ok (${(bytes.length / 1048576).toFixed(0)} MB)`); + + const archive = path.join(tmp, asset); + fs.writeFileSync(archive, bytes); + extract(archive, tmp); + return tmp; +} + +async function main() { + // `--target` exists for CI, which provisions for the runner it is on; without it + // the host is the target, which is what every local build wants. + const targetArg = process.argv.find((a) => a.startsWith("--target=")); + const tag = targetArg + ? targetArg.slice("--target=".length) + : `${process.platform}-${process.arch}`; + const [targetPlatform] = tag.split("-"); + + // Not an error, and deliberately exit 0: `build:mac` runs on an Intel runner for + // the x64 DMG, and there is no upstream library to give it. Failing here would + // break a release build over a feature that is designed to be absent gracefully. + if (tag === "darwin-x64") { + console.log( + "ONNX Runtime is not provisioned for darwin-x64: Microsoft publishes no\n" + + "osx-x86_64 (or universal) asset for 1.27 or later — arm64 is the only macOS\n" + + "target. The webcam background effects are therefore OFF on Intel Macs; the\n" + + "compositor logs one line and draws the camera unsegmented. Nothing else changes.", + ); + return; + } + + const spec = PINNED[tag]; + if (!spec) { + console.log( + `No pinned ONNX Runtime for ${tag} — skipping. Have: ${Object.keys(PINNED).join(", ")}`, + ); + return; + } + if (!MAGIC[targetPlatform]) { + throw new Error(`Unknown target platform in --target=${tag}`); + } + + const binDir = path.join(ROOT, "electron", "native", "bin", tag); + const dest = path.join(binDir, spec.out); + + if (fs.existsSync(dest) && !process.argv.includes("--force")) { + // Re-verify rather than trusting the filename: this directory is gitignored + // scratch space that a half-finished run or a hand copy can leave anything in. + const buf = fs.readFileSync(dest); + const magic = MAGIC[targetPlatform]; + const looksRight = + magic.bytes.every((b, i) => buf[i] === b) && + buf.includes(Buffer.from(`\0${VERSION}\0`, "latin1")); + if (looksRight) { + console.log(`Already present: ${dest}`); + console.log(` ONNX Runtime ${VERSION}, ${(buf.length / 1048576).toFixed(1)} MB`); + console.log("Use --force to re-download."); + return; + } + console.log(`Present but not ONNX Runtime ${VERSION} — re-fetching: ${dest}`); + } + + const tmp = await download(spec); + try { + const lib = find(tmp, spec.member); + if (!lib) throw new Error(`${spec.member} not found inside ${assetName(spec)}`); + const license = find(tmp, "LICENSE"); + if (!license) throw new Error(`LICENSE not found inside ${assetName(spec)}`); + + // Verify BEFORE vendoring: nothing unchecked reaches electron/native/bin, + // where the packager would happily ship it. + console.log("Verifying..."); + const banner = verify(lib, license, targetPlatform); + + fs.mkdirSync(binDir, { recursive: true }); + fs.copyFileSync(lib, dest); + if (targetPlatform !== "win32") fs.chmodSync(dest, 0o755); + + console.log(` ${banner}`); + // Où que viennent les octets, dire de quelle source ils sortent. Pour un artefact + // amont c'est le tag ; pour un que nous avons construit (`baseUrl`), c'est le commit + // que le workflow a compilé, et c'est la seule chose qui rend le binaire traçable + // une fois qu'il ne porte plus le nom de Microsoft. + if (spec.baseUrl) { + console.log(` built here from microsoft/onnxruntime@${SOURCE_COMMIT}`); + } + console.log(`\nVendored -> ${dest}`); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +main().catch((err) => { + console.error(`\n${err.message}`); + process.exit(1); +}); diff --git a/scripts/fetch-onnxruntime.test.mjs b/scripts/fetch-onnxruntime.test.mjs new file mode 100644 index 000000000..6424e4c7f --- /dev/null +++ b/scripts/fetch-onnxruntime.test.mjs @@ -0,0 +1,116 @@ +// The ONNX Runtime pin is coupled to a pin in a DIFFERENT LANGUAGE, and nothing +// else notices when they drift apart. +// +// `crates/Cargo.toml` gives `ort` the feature `api-NN`. That NN is the minimum ONNX +// Runtime minor version the crate accepts: `ort_sys` computes `ORT_API_VERSION` from +// it and asks the library for that API, and a library older than NN returns null — +// at which point `ort` PANICS rather than erroring (it is documented doing so in +// segmentation.rs, and it took down a render thread once already). So bumping `ort` +// without re-pinning this script ships a build where the effect is not merely off +// but actively fatal on the first frame that asks for it. +// +// The reverse drift is quieter and worse: pinning a NEWER runtime than the crate was +// built for makes `ort` log a compatibility warning to stderr and carry on, which in +// a packaged Electron app nobody reads. +// +// Neither direction is visible in review — the two lines are in different files, in +// different languages, edited by different tasks. This test is the thing that sees it. +// +// Read as source text rather than imported: fetch-onnxruntime.mjs calls main() at +// import and would start downloading. The property under test is a property of the +// literal table anyway. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const source = fs.readFileSync(path.join(HERE, "fetch-onnxruntime.mjs"), "utf8"); +const cargoToml = fs.readFileSync(path.join(HERE, "..", "crates", "Cargo.toml"), "utf8"); + +const version = source.match(/^const VERSION = "([^"]+)";/m)?.[1]; +// Anchored at the property so the file's (extensive) prose cannot match. +const digests = [...source.matchAll(/^\s*sha256:\s*"([^"]+)"/gm)].map((m) => m[1]); +const slugs = [...source.matchAll(/^\s*slug:\s*"([^"]+)"/gm)].map((m) => m[1]); +const tags = [...source.matchAll(/^\t"([a-z0-9]+-[a-z0-9]+)":\s*\{$/gm)].map((m) => m[1]); + +/** The `api-NN` feature `crates/Cargo.toml` gives `ort`, as a number. */ +const ortApiFloor = () => { + const block = cargoToml.match(/^ort = \{[\s\S]*?^\]\s*\}/m)?.[0] ?? ""; + const found = [...block.matchAll(/"api-(\d+)"/g)].map((m) => Number(m[1])); + return found.length ? Math.max(...found) : null; +}; + +describe("fetch-onnxruntime pins", () => { + // Without this, a reformat that breaks the regexes above would leave every other + // assertion iterating an empty array and passing vacuously. + it("still finds the pin table", () => { + expect(version, "VERSION not found in fetch-onnxruntime.mjs").toMatch(/^\d+\.\d+\.\d+$/); + expect(slugs.length).toBeGreaterThanOrEqual(4); + expect(digests).toHaveLength(slugs.length); + expect(tags).toHaveLength(slugs.length); + }); + + // THE point of this file. + it("pins a runtime that satisfies the `api-NN` floor in crates/Cargo.toml", () => { + const floor = ortApiFloor(); + expect(floor, "no api-NN feature found on `ort` in crates/Cargo.toml").toBeGreaterThan(0); + const minor = Number(version.split(".")[1]); + expect( + minor, + `crates/Cargo.toml asks ort for api-${floor}, so ONNX Runtime must be >= 1.${floor}.x, ` + + `but fetch-onnxruntime.mjs pins ${version}. A runtime below the floor makes GetApi ` + + "return null and ort PANICS. Re-pin VERSION and every sha256 together.", + ).toBeGreaterThanOrEqual(floor); + }); + + // Above the floor is not free either: ort warns at load and carries on, which in a + // packaged app goes to a stderr nobody reads. Exact match is the intended state. + it("pins the runtime the crate was actually built for, not merely a compatible one", () => { + const floor = ortApiFloor(); + expect( + Number(version.split(".")[1]), + `ONNX Runtime ${version} does not match the api-${floor} ort was built against. ` + + "Below it, ort panics; above it, ort logs a compatibility warning at every " + + "startup, into a stderr no packaged app shows. If the mismatch is deliberate, " + + "move ort to the matching api-NN feature in the same change.", + ).toBe(floor); + }); + + // The gpu_cuda variants are 200-320 MB and carry NVIDIA runtime redistribution + // terms. Nothing in this app uses a GPU execution provider — the CPU EP was the + // measured choice (webcam-segmentation.md), and it is what makes ONNX Runtime + // shippable at all. + it("pins only the plain CPU assets", () => { + for (const slug of slugs) { + expect(slug, `${slug} is not a plain CPU asset`).not.toMatch( + /gpu|cuda|tensorrt|qnn|training/, + ); + } + }); + + it("pins a full sha-256 for every asset", () => { + for (const digest of digests) { + expect(digest).toMatch(/^[0-9a-f]{64}$/); + } + }); + + // The keys are matched against `${process.platform}-${process.arch}`, so a + // plausible-looking typo (`darwin-aarch64`, `win-x64`) silently provisions + // nothing and the effect is off with no error anywhere. + it("keys the table by real process.platform-process.arch tags", () => { + for (const tag of tags) { + expect(tag).toMatch(/^(win32|darwin|linux)-(x64|arm64)$/); + } + expect(new Set(tags).size, "duplicate tag in the pin table").toBe(tags.length); + }); + + // Microsoft publishes no osx-x86_64 asset from 1.27 on. main() has a branch that + // explains that and exits 0 so the Intel release build still succeeds; if someone + // adds a darwin-x64 entry, that branch makes it dead and the effect stays off. + it("does not pin darwin-x64, which upstream does not publish", () => { + expect(tags).not.toContain("darwin-x64"); + expect(source).toMatch(/tag === "darwin-x64"/); + }); +}); diff --git a/scripts/ffmpeg-linked-libraries.test.mjs b/scripts/ffmpeg-linked-libraries.test.mjs new file mode 100644 index 000000000..49e0bd0d3 --- /dev/null +++ b/scripts/ffmpeg-linked-libraries.test.mjs @@ -0,0 +1,111 @@ +// `crates/compositor/build.rs` decides which ffmpeg shared libraries the native addon +// imports. Six places have to agree with that list, and none of them is derived from it: +// +// - scripts/fetch-ffmpeg.mjs vendors the Windows DLLs and decides when to skip +// - scripts/before-pack.cjs fails the pack if one is missing (three OS tables) +// - scripts/build-linux-compositor-addon.mjs copies + symbol-renames the sonames +// - nix/compositor-view.nix builds symbols.map from a brace glob +// +// Drift is not a cosmetic problem. The addon is a cdylib, so a missing library does not +// fail the link: it fails at `require()` with "undefined symbol: osff_avfilter_graph_alloc", +// `compositorViewService` logs "native addon not present; running as no-op", and the app +// ships with a blank preview and every export dead — the exact symptom 1.9.0 shipped with. +// Every guard listed above passes in that state, because each one only knows the list it +// was written with. +// +// This test derives the truth from build.rs and checks the other five against it, so +// linking a seventh library fails here instead of in a user's installer. It reads source +// text: before-pack.cjs and fetch-ffmpeg.mjs both do work at import time, and the property +// under test is a property of the literal lists anyway. +// +// The nix derivation is the reason this file exists rather than an assertion inside +// before-pack.test.mjs: `.github/workflows/nix-build.yml` does not run on pull requests +// and there is no nix on the Windows dev box, so nix/compositor-view.nix reaches main with +// no pre-merge signal at all. A text assertion is not `nix build`, but it does catch the +// one mistake that has actually happened. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); +const read = (relative) => fs.readFileSync(path.join(repoRoot, relative), "utf8"); + +const buildRs = read("crates/compositor/build.rs"); + +/** The `for lib in [...] { println!("cargo:rustc-link-lib=…") }` list, in build.rs order. */ +const linked = (() => { + const block = buildRs.match(/for lib in \[([\s\S]*?)\]\s*\{[\s\S]*?rustc-link-lib/); + if (!block) return []; + return [...block[1].matchAll(/"([a-z]+)"/g)].map((match) => match[1]); +})(); + +describe("the ffmpeg libraries the compositor links", () => { + // Without this the regex could silently match nothing and every assertion below + // would iterate an empty list and pass vacuously. + it("is read out of build.rs", () => { + expect(linked.length).toBeGreaterThanOrEqual(6); + expect(linked).toContain("avcodec"); + expect(linked).toContain("avfilter"); + }); + + it("is vendored in full by fetch-ffmpeg.mjs", () => { + // The probe that decides whether a warm tree still needs a re-vendor. When it + // listed fewer libraries than build.rs links, a tree holding the previous set + // satisfied it and the new DLL was never fetched. + const source = read("scripts/fetch-ffmpeg.mjs"); + const table = source.match(/REQUIRED_SHARED_DLLS = \[([\s\S]*?)\]/); + expect(table, "fetch-ffmpeg.mjs no longer declares REQUIRED_SHARED_DLLS").not.toBeNull(); + const required = [...table[1].matchAll(/"([a-z]+)"/g)].map((match) => match[1]); + expect([...required].sort()).toEqual([...linked].sort()); + }); + + it("is staged and symbol-renamed for the Linux addon", () => { + const source = read("scripts/build-linux-compositor-addon.mjs"); + const table = source.match(/FFMPEG_SONAMES = \[([\s\S]*?)\]/); + expect( + table, + "build-linux-compositor-addon.mjs no longer declares FFMPEG_SONAMES", + ).not.toBeNull(); + const sonames = [...table[1].matchAll(/"lib([a-z]+)\.so\.\d+"/g)].map((match) => match[1]); + expect([...sonames].sort()).toEqual([...linked].sort()); + }); + + it("is staged and symbol-renamed by the nix derivation", () => { + // `for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,…}.so.*` — the brace glob feeds + // both the copy into $stage/lib and the symbols.map the addon is linked against. + // A name missing here links against nixpkgs' un-renamed copy, and the + // installPhase leak check only flags symbols WITHOUT the osff_ prefix, so it + // passes either way. + const source = read("nix/compositor-view.nix"); + const glob = source.match(/\/lib\/lib\{([a-z,]+)\}\.so\.\*/); + expect(glob, "nix/compositor-view.nix no longer globs the ffmpeg sonames").not.toBeNull(); + expect(glob[1].split(",").sort()).toEqual([...linked].sort()); + }); + + describe("is required by before-pack.cjs", () => { + const source = read("scripts/before-pack.cjs"); + /** The `[...]` array literal a `...[…].map(` spread iterates, per OS table. */ + const listsIn = (table) => { + const block = source.slice(source.indexOf(`const ${table} = [`)); + const end = block.indexOf("\n];"); + return [...block.slice(0, end).matchAll(/\.\.\.\[([^\]]*)\]\.map\(/g)].flatMap((match) => + [...match[1].matchAll(/"([a-z]+)"/g)].map((name) => name[1]), + ); + }; + + // One requirement per library rather than `atLeast: N` over a combined regex: + // several versioned copies of one library would satisfy a count while another + // was missing entirely, which is how a broken pack passed the guard before. + for (const table of ["MAC_REQUIRED", "LINUX_REQUIRED", "WIN_REQUIRED"]) { + it(table, () => { + const required = listsIn(table); + expect(required.length, `${table} declares no per-library spread`).toBeGreaterThan(0); + for (const library of linked) { + expect(required, `${table} does not require lib${library}`).toContain(library); + } + }); + } + }); +}); diff --git a/scripts/macos-floor.mjs b/scripts/macos-floor.mjs new file mode 100644 index 000000000..347685bc6 --- /dev/null +++ b/scripts/macos-floor.mjs @@ -0,0 +1,80 @@ +// Reads the macOS floor the app declares to LaunchServices out of electron-builder.json5. +// +// Its own module because the number is asserted from two different guards — the +// Package.swift floor check and before-pack's pack-time payload check — and a second copy +// of the parser is precisely how one of them ends up hardened and the other not. That +// already happened once: declaredMacOsFloor() was scoped to its declaration block after +// review, while the function written directly beside it still took the first match in the +// whole file. +// +// Hand-rolled rather than a JSON5 parse to stay dependency-free and runnable on every CI +// platform. + +/** + * Drops `//` comments, ignoring any that appear inside a string — electron-builder.json5 + * is heavily commented, and its comments discuss the very keys parsed below. + * + * String-aware rather than a plain `s.replace(/\/\/.*$/gm, "")` because the config also + * carries URLs, whose `//` a naive strip would eat. + */ +function stripJson5Comments(source) { + let out = ""; + let inString = false; + for (let i = 0; i < source.length; i++) { + const ch = source[i]; + if (inString) { + out += ch; + if (ch === "\\") { + out += source[++i] ?? ""; + } else if (ch === '"') { + inString = false; + } + continue; + } + if (ch === '"') { + inString = true; + out += ch; + continue; + } + if (ch === "/" && source[i + 1] === "/") { + while (i < source.length && source[i] !== "\n") i++; + out += "\n"; + continue; + } + out += ch; + } + return out; +} + +/** The body of a top-level `"": { ... }` object, brace-matched. */ +function objectBody(source, key) { + const opener = new RegExp(`"${key}"\\s*:\\s*{`).exec(source); + if (!opener) { + return null; + } + let depth = 0; + for (let i = opener.index + opener[0].length - 1; i < source.length; i++) { + if (source[i] === "{") depth++; + else if (source[i] === "}" && --depth === 0) { + return source.slice(opener.index + opener[0].length, i); + } + } + return null; +} + +/** The declared macOS major floor, or null if the `mac` block does not carry one. */ +export function declaredAppFloorFrom(source) { + const mac = objectBody(stripJson5Comments(source), "mac"); + if (!mac) { + return null; + } + const match = mac.match(/"minimumSystemVersion"\s*:\s*"(\d+)(?:\.\d+)*"/); + return match ? Number(match[1]) : null; +} + +/** The full declared version string (e.g. "13.0"), for callers comparing exactly. */ +export function declaredAppVersionFrom(source) { + const mac = objectBody(stripJson5Comments(source), "mac"); + const match = mac?.match(/"minimumSystemVersion"\s*:\s*"([\d.]+)"/); + return match ? match[1] : null; +} diff --git a/scripts/stage-vcomp-runtime.mjs b/scripts/stage-vcomp-runtime.mjs index 06b35d6cf..d987946ea 100644 --- a/scripts/stage-vcomp-runtime.mjs +++ b/scripts/stage-vcomp-runtime.mjs @@ -1,4 +1,6 @@ -// Stages vcomp140.dll beside the whisper/ggml payload it is loaded by. +// Stages the Visual C++ runtime DLLs that the prebuilt payload imports, beside it. +// +// Two independent binaries need this, for the same reason and with the same fix. // // ggml-base.dll and ggml-cpu.dll are compiled with OpenMP, so they import // vcomp140.dll — Microsoft's OpenMP runtime, which ships with the Visual C++ @@ -8,16 +10,26 @@ // and transcription fail with the unactionable timeout described in // scripts/before-pack.cjs. // +// onnxruntime.dll — vendored by scripts/fetch-onnxruntime.mjs for the camera +// background segmentation — imports the CRT proper: msvcp140, msvcp140_1, +// vcruntime140, vcruntime140_1. It arrives as an upstream release binary, so +// `-C target-feature=+crt-static` is not available the way it is for our own Rust +// addon; the only remedy left is the one before-pack names, which is this file. +// Without it `checkWinNoRedistDependency` refuses to pack at all, and the Windows +// installer cannot be built — while `WIN_REQUIRED` in the same hook refuses to pack +// *without* onnxruntime.dll, so dropping it is not an escape either. +// // This is the same class of failure that Store certification rejected 1.9.1 for, // and it survived that fix because the guard only looked for msvcp/vcruntime/concrt // prefixes — `vcomp` matches none of them. The guard now covers the whole family // and, more usefully, only objects when the DLL is not shipped alongside. // -// Shipping the DLL rather than rebuilding whisper without OpenMP is deliberate: +// Shipping the DLLs rather than rebuilding without them is deliberate. For whisper // it leaves the computation byte-for-byte identical, where -DGGML_OPENMP=OFF would // swap OpenMP's scheduler for ggml's own and change transcription throughput by an -// amount nobody has measured. 200 KB against that unknown is a cheap trade. If the -// dependency ever becomes inconvenient, measure first, then switch. +// amount nobody has measured. For ONNX Runtime there is nothing to rebuild. ~1 MB +// against that is a cheap trade. If the dependency ever becomes inconvenient, +// measure first, then switch. import fs from "node:fs"; import path from "node:path"; @@ -27,10 +39,19 @@ import { findVcVarsAll } from "./msvcEnv.mjs"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.join(__dirname, ".."); const DEST_DIR = path.join(ROOT, "electron", "native", "bin", "win32-x64"); -const DLL = "vcomp140.dll"; +// Lower-case, because that is how they are compared against `readdirSync` names. +// vcomp140 lives in Microsoft.VC.OpenMP, the other four in Microsoft.VC.CRT — +// sibling directories under the same Redist tree, so one walk finds them all. +const DLLS = [ + "vcomp140.dll", + "msvcp140.dll", + "msvcp140_1.dll", + "vcruntime140.dll", + "vcruntime140_1.dll", +]; if (process.platform !== "win32") { - console.log("Skipping OpenMP runtime staging: Windows-only."); + console.log("Skipping Visual C++ runtime staging: Windows-only."); process.exit(0); } @@ -65,8 +86,11 @@ function searchRoots() { ]; } +/** Candidate paths per DLL name, from ONE walk — the trees are large enough that + * walking them once per name would be the slowest part of the build. */ function findRedistCopies() { - const found = []; + const wanted = new Set(DLLS); + const found = new Map(DLLS.map((name) => [name, []])); const walk = (dir, depth) => { if (depth > 8) return; let entries; @@ -77,16 +101,13 @@ function findRedistCopies() { } for (const entry of entries) { const full = path.join(dir, entry.name); + const lower = entry.name.toLowerCase(); if (entry.isDirectory()) { walk(full, depth + 1); - } else if ( - entry.name.toLowerCase() === DLL && - /\\Redist\\/i.test(full) && - /\\x64\\/i.test(full) - ) { + } else if (wanted.has(lower) && /\\Redist\\/i.test(full) && /\\x64\\/i.test(full)) { // `onecore\x64` is a trimmed variant for Windows Core headless SKUs; the // desktop app wants the ordinary one. - if (!/\\onecore\\/i.test(full)) found.push(full); + if (!/\\onecore\\/i.test(full)) found.get(lower).push(full); } } }; @@ -94,35 +115,42 @@ function findRedistCopies() { return found; } -const candidates = findRedistCopies(); -if (candidates.length === 0) { - throw new Error( - `Could not find a redistributable ${DLL} under any Visual Studio installation.\n\n` + - "It lives in VC\\Redist\\MSVC\\\\x64\\Microsoft.VC.OpenMP\\.\n" + - "Install the Visual Studio C++ workload, which is required to build the native\n" + - "helpers anyway. Without this file the shipped whisper/ggml libraries cannot load\n" + - "on a machine that has no Visual C++ Redistributable, and transcription fails there\n" + - "with no usable error.", - ); -} - // Newest by file version, so a machine carrying several toolsets stages the latest. const versionOf = (file) => { const match = file.match(/MSVC\\(\d+(?:\.\d+)*)\\/i); return match ? match[1].split(".").map(Number) : [0]; }; -candidates.sort((a, b) => { +const newestFirst = (a, b) => { const [x, y] = [versionOf(a), versionOf(b)]; for (let i = 0; i < Math.max(x.length, y.length); i++) { if ((x[i] ?? 0) !== (y[i] ?? 0)) return (y[i] ?? 0) - (x[i] ?? 0); } return 0; -}); +}; -const source = candidates[0]; -fs.mkdirSync(DEST_DIR, { recursive: true }); -const dest = path.join(DEST_DIR, DLL); -fs.copyFileSync(source, dest); +const copies = findRedistCopies(); + +// Report every missing name at once. Staging four of five and failing on the fifth +// would send someone back through the same install-and-retry loop per DLL. +const missing = DLLS.filter((name) => copies.get(name).length === 0); +if (missing.length > 0) { + throw new Error( + `Could not find a redistributable ${missing.join(", ")} under any Visual Studio installation.\n\n` + + "They live in VC\\Redist\\MSVC\\\\x64\\ — vcomp140.dll under\n" + + "Microsoft.VC.OpenMP, the rest under Microsoft.VC.CRT.\n" + + "Install the Visual Studio C++ workload, which is required to build the native\n" + + "helpers anyway. Without these files the shipped whisper/ggml libraries and the\n" + + "ONNX Runtime cannot load on a machine that has no Visual C++ Redistributable:\n" + + "transcription fails there with no usable error, and the camera background is\n" + + "silently inert. before-pack refuses to package either way.", + ); +} -console.log(`Staged ${DLL} from ${source}`); -console.log(` -> ${path.relative(ROOT, dest)}`); +fs.mkdirSync(DEST_DIR, { recursive: true }); +for (const name of DLLS) { + const source = copies.get(name).sort(newestFirst)[0]; + const dest = path.join(DEST_DIR, name); + fs.copyFileSync(source, dest); + console.log(`Staged ${name} from ${source}`); + console.log(` -> ${path.relative(ROOT, dest)}`); +} diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs index e1dc48148..785e48c1e 100644 --- a/scripts/test-windows-wgc-helper.mjs +++ b/scripts/test-windows-wgc-helper.mjs @@ -45,6 +45,34 @@ const WITH_STALLED_READBACK = process.env.OPENSCREEN_WGC_TEST_STALL_READBACK === "true" || process.argv.includes("--stall-readback"); const STALL_READBACK_MS = Number(process.env[STALL_READBACK_ENV] ?? 60_000); +const STALL_FRAME_CALLBACK_ENV = "OPENSCREEN_WGC_TEST_STALL_FRAME_CALLBACK_MS"; +/** + * Reproduces getopenscreen/openscreen#460 on ordinary hardware: stalls the WGC + * frame *callback* itself while it holds the frame lock, the shape that issue + * actually reproduced on Intel HD 520 ("A WGC frame callback did not finish"). + * Distinct from WITH_STALLED_READBACK above -- that stalls the writer's own + * readback, which quiesceLegacyCallback()'s drain cannot see + * (callbacksInFlight_ stays at zero), so it cannot exercise the + * video-writer-join skip this stall exists to test. + * + * Forces the legacy push path on (below), because that is the only path with a + * frame callback to stall: the default pull path has no WGC-owned thread, so + * this scenario would otherwise stall nothing and assert on a skip that can + * never be taken. + */ +const WITH_STALLED_FRAME_CALLBACK = + process.env.OPENSCREEN_WGC_TEST_STALL_FRAME_CALLBACK === "true" || + process.argv.includes("--stall-frame-callback"); +const STALL_FRAME_CALLBACK_MS = Number(process.env[STALL_FRAME_CALLBACK_ENV] ?? 60_000); +const LEGACY_FRAME_CALLBACK_ENV = "OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK"; +/** + * Runs any scenario on the pre-#306 push-based delivery path instead of the + * pull-based default -- the same lever a user gets, so a machine that only + * fails one way can be A/B'd without swapping builds. + */ +const WITH_LEGACY_FRAME_CALLBACK = + process.env.OPENSCREEN_WGC_LEGACY_FRAME_CALLBACK === "1" || + process.argv.includes("--legacy-frame-callback"); const STOP_BUDGET_ENV = "OPENSCREEN_WGC_STOP_BUDGET_MS"; /** * The helper's global shutdown ceiling, pinned into its environment below so @@ -64,18 +92,34 @@ if (WITH_SOFTWARE_ENCODER && WITH_SOFTWARE_FALLBACK) { throw new Error("--software-encoder and --software-fallback are mutually exclusive"); } -function runHelper(config, { injectDefaultSinkWriterFailure = false, stallReadbackMs = 0 } = {}) { +function runHelper( + config, + { + injectDefaultSinkWriterFailure = false, + stallReadbackMs = 0, + stallFrameCallbackMs = 0, + legacyFrameCallback = false, + } = {}, +) { return new Promise((resolve, reject) => { const env = { ...process.env }; delete env[INJECT_DEFAULT_SINK_WRITER_FAILURE_ENV]; delete env[STALL_READBACK_ENV]; + delete env[STALL_FRAME_CALLBACK_ENV]; + delete env[LEGACY_FRAME_CALLBACK_ENV]; env[STOP_BUDGET_ENV] = String(STOP_BUDGET_MS); + if (legacyFrameCallback) { + env[LEGACY_FRAME_CALLBACK_ENV] = "1"; + } if (injectDefaultSinkWriterFailure) { env[INJECT_DEFAULT_SINK_WRITER_FAILURE_ENV] = "1"; } if (stallReadbackMs > 0) { env[STALL_READBACK_ENV] = String(stallReadbackMs); } + if (stallFrameCallbackMs > 0) { + env[STALL_FRAME_CALLBACK_ENV] = String(stallFrameCallbackMs); + } const child = spawn(HELPER_PATH, [JSON.stringify(config)], { env, stdio: ["pipe", "pipe", "pipe"], @@ -213,6 +257,48 @@ function startFixtureWindow() { }); } +/** + * Windows Graphics Capture delivers frames on compositor damage, not on a + * fixed clock -- on a genuinely idle desktop the frame pool can go a full + * test run without ever firing FrameArrived once. That is invisible to most + * of this harness, which just needs *a* frame eventually, but the + * stalled-frame-callback regression check needs one to land *inside* the + * DURATION_MS window specifically, so the stall this injects is actually the + * thing holding the frame lock when `stop` arrives. + * + * Moving the cursor alone does not reliably do this: most modern GPU/driver + * combinations composite the cursor on its own hardware overlay plane, so + * repositioning it never touches the desktop bitmap WGC captures (confirmed + * empirically here -- frames=0 with a cursor-only nudge running the whole + * test). A visible window changing position is not optional the way the + * cursor is; DWM has to redraw the area it moved across. Returns a stop + * function; always call it, paired failure or not, or the window and its + * PowerShell host outlive the test process. + */ +function startScreenActivity() { + const child = spawn( + "powershell", + [ + "-NoProfile", + "-Command", + "Add-Type -AssemblyName System.Windows.Forms; " + + "$f = New-Object System.Windows.Forms.Form; " + + "$f.StartPosition = 'Manual'; $f.Location = New-Object System.Drawing.Point(0,0); " + + "$f.Size = New-Object System.Drawing.Size(200,200); " + + "$f.TopMost = $true; $f.Show(); " + + "$x = 0; " + + "while ($true) { " + + "$f.Location = New-Object System.Drawing.Point($x, 0); " + + "$x = ($x + 20) % 200; " + + "[System.Windows.Forms.Application]::DoEvents(); " + + "Start-Sleep -Milliseconds 100; " + + "}", + ], + { stdio: ["ignore", "ignore", "ignore"], windowsHide: false }, + ); + return () => child.kill(); +} + function normalizeDeviceName(value) { return value .toLowerCase() @@ -413,16 +499,20 @@ const config = { }, }; +const stopScreenActivity = WITH_STALLED_FRAME_CALLBACK ? startScreenActivity() : null; let result; try { result = await runHelper(config, { injectDefaultSinkWriterFailure: WITH_SOFTWARE_FALLBACK, stallReadbackMs: WITH_STALLED_READBACK ? STALL_READBACK_MS : 0, + stallFrameCallbackMs: WITH_STALLED_FRAME_CALLBACK ? STALL_FRAME_CALLBACK_MS : 0, + legacyFrameCallback: WITH_LEGACY_FRAME_CALLBACK || WITH_STALLED_FRAME_CALLBACK, }); } finally { if (fixtureWindow) { fixtureWindow.child.kill(); } + stopScreenActivity?.(); } // The regression check for issue #252. With the frame lock deliberately wedged @@ -453,6 +543,51 @@ if (WITH_STALLED_READBACK) { process.exit(0); } +// The regression check for getopenscreen/openscreen#460: a frame callback +// wedged inside the driver, confirmed on real hardware via a Save Diagnostics +// report. Before the fix, video-writer-join burned its whole step budget +// joining a thread parked behind that same stuck callback -- this asserts +// both that the helper still exits promptly (not the ~13s that step's own +// budget alone would cost) and that it took the specific skip path rather +// than any other route to exiting. +if (WITH_STALLED_FRAME_CALLBACK) { + if (result.stopHung) { + throw new Error( + `Helper survived ${STOP_HANG_LIMIT_MS}ms past "stop" with a stalled frame callback. ` + + "Its shutdown watchdog did not fire (issue #460).", + ); + } + const steps = readStopTimingSteps(result.stderr); + if (!steps.includes("command-received")) { + throw new Error(`Helper never acknowledged "stop". Steps seen: ${steps.join(", ") || "none"}`); + } + if (!result.stderr.includes("reason=frame-callback-stuck")) { + throw new Error( + `Helper did not take the video-writer-join skip path. stderr:\n${result.stderr}`, + ); + } + // wgc-quiesce's own drain is a fixed 5000ms, so a healthy skip lands + // there plus the near-instant audio/microphone/webcam steps -- nowhere + // near the ~13s (5s drain + the 8s step budget) the join it replaces + // would have cost before this fix. + const STALLED_FRAME_CALLBACK_LATENCY_BUDGET_MS = 10_000; + if ( + result.stopLatencyMs !== null && + result.stopLatencyMs > STALLED_FRAME_CALLBACK_LATENCY_BUDGET_MS + ) { + throw new Error( + `Stop took ${result.stopLatencyMs}ms with a stalled frame callback, over the ` + + `${STALLED_FRAME_CALLBACK_LATENCY_BUDGET_MS}ms budget the video-writer-join skip should keep it under.`, + ); + } + console.log("WGC helper stalled-frame-callback stop check passed", { + stopLatencyMs: result.stopLatencyMs, + steps, + }); + fs.rmSync(outputPath, { force: true }); + process.exit(0); +} + assertStopWasClean(result); if (result.code !== 0) { @@ -554,6 +689,34 @@ if ( `WGC helper encoder selection was ${JSON.stringify(encoderSelection)}, expected ${expectedEncoderSelection} with preferSoftwareEncoder=${WITH_SOFTWARE_ENCODER}: ${result.stdout}`, ); } +// videoEncoderRuntime is separate from `video` above: it is what +// GetTransformForStream found in the sink writer's own resolved pipeline +// after BeginWriting(), not which configuration path was tried. "unknown" +// here on a run that otherwise passed means the introspection itself is +// broken (wrong COM call, wrong category, wrong attribute), not a real +// ambiguity -- a healthy sink writer always has exactly one encoder node. +if (!["hardware", "software", "unknown"].includes(encoderSelection.videoEncoderRuntime)) { + throw new Error( + `WGC helper reported an unrecognised videoEncoderRuntime: ${JSON.stringify(encoderSelection)}`, + ); +} +if (encoderSelection.videoEncoderRuntime === "unknown") { + throw new Error( + `WGC helper could not introspect its own sink writer for videoEncoderRuntime: ${JSON.stringify(encoderSelection)}`, + ); +} +// forceSoftwareEncoder disables hardware transforms explicitly +// (MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS=FALSE), so this is deterministic +// regardless of what the test machine has registered -- unlike the "default" +// path, whose runtime legitimately depends on the machine. +if ( + (WITH_SOFTWARE_ENCODER || WITH_SOFTWARE_FALLBACK) && + encoderSelection.videoEncoderRuntime !== "software" +) { + throw new Error( + `WGC helper forced the software encoder but videoEncoderRuntime was ${encoderSelection.videoEncoderRuntime}, expected software: ${JSON.stringify(encoderSelection)}`, + ); +} // Every fallback path has to stay fragmented, not just the nominal one. The // helper degrades to the plain container rather than failing a recording, so // without this the fix could quietly stop applying and every other assertion diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 31f12233e..9a7f1b590 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -30,6 +30,7 @@ import { buildAutoZoomSuggestions } from "@/lib/ai-edition/timeline/zoom-suggest import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; import { GIF_SIZE_PRESETS, type GifSizePreset } from "@/lib/exporter"; import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; +import { outputFrameCount } from "@/lib/exporter/outputFrameCount"; import { mixVoiceoverIntoVideo } from "@/lib/exporter/voiceoverMix"; import { exportGifNative, exportMultiNative, nativeBridgeClient } from "@/native"; import type { CompositorClipInput } from "@/native/contracts"; @@ -263,22 +264,23 @@ async function runExport(request: CliExportRequest): Promise { aspectRatioValue, }); - const clips = buildNativeClipList(axcutDocument); - if (clips.length === 0) { + const builtClips = buildNativeClipList(axcutDocument); + if (builtClips.length === 0) { throw new Error("The project's timeline has no visible clips to export"); } - const sceneJson = JSON.stringify(buildSceneDescription(axcutDocument)); + const sceneDesc = buildSceneDescription(axcutDocument); + + // The webcam background effect is applied by the compositor from the scene, so the clip + // list needs no pre-rendering pass. + const clips = builtClips; + const sceneJson = JSON.stringify(sceneDesc); // Progress: native pushes raw encoded-frame counts; totals and pacing are // computed here, mirroring the ExportDialog. const outFps = format === "gif" ? gifFrameRate : MP4_EXPORT_FPS; - const totalFrames = Math.max( - 1, - Math.round( - clips.reduce((sum, clip) => sum + Math.max(0, clip.sourceEndSec - clip.sourceStartSec), 0) * - outFps, - ), - ); + // Speed-adjusted, not source seconds — see `outputFrameCount`. Counting raw duration + // is what made a 1.25x timeline stop the bar at 80% (OpenScreen#371). + const totalFrames = outputFrameCount(clips, sceneDesc.speedRegions, outFps); const exportStartedAt = Date.now(); const unsubscribeProgress = window.electronAPI.onNativeExportProgress?.((frames: number) => { const elapsedSec = (Date.now() - exportStartedAt) / 1000; diff --git a/src/components/ai-edition/AudioTrackPane.test.tsx b/src/components/ai-edition/AudioTrackPane.test.tsx new file mode 100644 index 000000000..010888217 --- /dev/null +++ b/src/components/ai-edition/AudioTrackPane.test.tsx @@ -0,0 +1,69 @@ +// @vitest-environment jsdom +import "@testing-library/jest-dom"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: (scope: string) => (key: string) => `${scope}.${key}`, +})); + +import { AudioTrackPane } from "./RightPanes"; + +describe("AudioTrackPane", () => { + const createTl = () => ({ + selectedAudioTrackId: "track_1", + audioTracks: [ + { + id: "track_1", + clipId: "clip_1", + assetId: "asset_audio_1", + startSec: 0, + durationSec: 5, + gainDb: 0, + fadeInMs: 0, + fadeOutMs: 0, + offsetSec: 0, + }, + ], + assets: [ + { + id: "asset_audio_1", + kind: "audio" as const, + label: "voice.mp3", + originalPath: "/path/voice.mp3", + durationSec: 5, + }, + ], + clearSelection: vi.fn(), + selectAudioTrack: vi.fn(), + setAudioTrackGain: vi.fn(), + setAudioTrackFade: vi.fn(), + removeAudioTrack: vi.fn(), + }); + + it("renders close button and closes audio track by default", () => { + const tl = createTl(); + render(); + + const closeBtn = screen.getByRole("button", { name: "common.actions.close" }); + expect(closeBtn).toBeInTheDocument(); + const header = closeBtn.closest("header"); + expect(header).toHaveStyle({ paddingRight: "var(--sp-4)" }); + const svg = closeBtn.querySelector("svg"); + expect(svg?.classList.contains("lucide-x")).toBe(true); + + fireEvent.click(closeBtn); + expect(tl.clearSelection).toHaveBeenCalledTimes(1); + }); + + it("calls custom onClose if provided", () => { + const tl = createTl(); + const onClose = vi.fn(); + render(); + + const closeBtn = screen.getByRole("button", { name: "common.actions.close" }); + fireEvent.click(closeBtn); + expect(onClose).toHaveBeenCalledTimes(1); + expect(tl.selectAudioTrack).not.toHaveBeenCalled(); + }); +}); diff --git a/src/components/ai-edition/CaptionsPane.gating.test.tsx b/src/components/ai-edition/CaptionsPane.gating.test.tsx index cae76b59a..5eee1b595 100644 --- a/src/components/ai-edition/CaptionsPane.gating.test.tsx +++ b/src/components/ai-edition/CaptionsPane.gating.test.tsx @@ -1,9 +1,12 @@ // @vitest-environment jsdom -// Captions are a view of the transcript, so the pane's "Transcribe video" -// button is a retry, not a first step — the background pass has already tried. -// On a media with no audio track that retry can only fail again, so the button -// has to be dead and the pane has to say what is wrong instead of inviting a -// pointless click. +// Captions are a view of the transcript, and since issue #560 they are reached from +// the transcript tab rather than owning one. So this pane no longer STARTS a +// transcription — the transcript tab's empty state carries the single gate. Two +// buttons for one background pass is what made people believe captions were +// transcribed separately. +// +// What the pane still owes the reader is a status: whether a pass is already +// running, and why there will never be one on a media with no audio track. import "@testing-library/jest-dom"; import { cleanup, render, screen } from "@testing-library/react"; @@ -76,6 +79,14 @@ function load(document: AxcutDocument) { }); } +function mount() { + render( + + + , + ); +} + beforeEach(() => { useTranscriptionStore.getState().reset(); useProjectStore.getState().clear(); @@ -86,43 +97,58 @@ afterEach(() => { }); describe("captions pane gating", () => { - it("offers the retry while the media might still yield a transcript", () => { + it("does not offer a second way to start a transcription", () => { load(documentWith(ASSET)); - render( - - - , - ); - expect(screen.getByRole("button", { name: "Transcribe video" })).toBeEnabled(); + mount(); + expect(screen.queryByRole("button", { name: "Transcribe video" })).toBeNull(); + expect( + screen.getByText("Captions are read from the media transcript.", { exact: false }), + ).toBeInTheDocument(); }); - it("shows the queued background run instead of an idle button", () => { + it("reports a background run that is already going", () => { load(documentWith(ASSET)); useTranscriptionStore.setState({ projectId: "proj_1", jobs: { asset_1: { status: "running", language: "auto", manual: false } }, }); - render( - - - , - ); - expect(screen.getByRole("button", { name: "Transcribing…" })).toBeDisabled(); + mount(); + expect(screen.getByText("Transcribing")).toBeInTheDocument(); + // Still not a control: a running pass is news, not something to press. + expect(screen.queryByRole("button", { name: "Transcribing" })).toBeNull(); }); - it("kills the retry on a media with no audio track and explains it", () => { + it("stays quiet when only an off-timeline asset is busy", () => { + // The gate answers for the timeline's assets; the label must not answer for the whole + // bin. A bin asset mid-transcription used to relabel the pane as if the film were + // being transcribed. + const offTimeline: AxcutAsset = { + id: "asset_2", + kind: "video", + label: "bin-only.mp4", + originalPath: "/bin.mp4", + durationSec: 8, + cameraTrack: null, + }; + const document = documentWith(ASSET); + document.assets.push(offTimeline); + load(document); + useTranscriptionStore.setState({ + projectId: "proj_1", + jobs: { asset_2: { status: "running", language: "auto", manual: false } }, + }); + mount(); + expect(screen.queryByText("Transcribing")).toBeNull(); + }); + + it("explains a media with no audio track, where no pass will ever help", () => { load( documentWith({ ...ASSET, transcriptionFailure: { kind: "no-audio", message: "No audio track found in this video." }, }), ); - render( - - - , - ); - expect(screen.getByRole("button", { name: "Transcribe video" })).toBeDisabled(); + mount(); expect( screen.getByText("This media has no audio track — there is nothing to transcribe."), ).toBeInTheDocument(); diff --git a/src/components/ai-edition/CaptionsPane.tsx b/src/components/ai-edition/CaptionsPane.tsx index 6d6a776c9..97276bcd1 100644 --- a/src/components/ai-edition/CaptionsPane.tsx +++ b/src/components/ai-edition/CaptionsPane.tsx @@ -9,7 +9,7 @@ // translation is stored beside the transcript, keyed by segment id, and picking // "Original" goes straight back to the SSOT text. -import { Captions as CaptionsIcon, Languages, Loader2, Trash2 } from "lucide-react"; +import { Captions as CaptionsIcon, Languages, Loader2, Trash2, X } from "lucide-react"; import { useMemo, useState } from "react"; import { useScopedT } from "@/contexts/I18nContext"; import type { CaptionAnchorH, CaptionAnchorV } from "@/lib/ai-edition/captions"; @@ -20,14 +20,17 @@ import { } from "@/lib/ai-edition/captions"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { + useAssetTranscriptions, useTimelineTranscriptGate, - useTranscriptionStore, } from "@/lib/ai-edition/store/transcriptionStore"; import { useCaptions } from "@/lib/ai-edition/store/useCaptions"; +import { firstTimelineBusyView } from "@/lib/ai-edition/transcription/status"; import { nativeBridgeClient } from "@/native"; import { ColorField } from "./ColorField"; import styles from "./NewEditorShell.module.css"; import { SliderCell, Toggle } from "./RightPanes"; +import { useTranscriptionLabel } from "./TranscriptionStatus"; +import { transcriptionBusyLabel } from "./transcriptionBusyLabel"; /** The families `src/index.css` already loads for on-canvas text — anything else * would render in the preview but fall back to a default in the export canvas. */ @@ -70,9 +73,10 @@ const TRANSLATION_LANGUAGES: ReadonlyArray<{ code: string; label: string }> = [ { code: "zh", label: "中文" }, ]; -export function CaptionsPane() { +export function CaptionsPane({ onClose }: { onClose?: () => void } = {}) { const t = useScopedT("settings"); const te = useScopedT("editor"); + const tc = useScopedT("common"); const { settings, translations, @@ -90,15 +94,25 @@ export function CaptionsPane() { // Captions are a view of the transcript, and the transcript arrives on its // own (transcriptionStore's background pass). The pane reads that state // straight from the store rather than being handed a busy flag: it is the - // same answer everywhere, and "Transcribe" here is only ever a retry. + // same answer everywhere, and this pane only ever reports on the pass — + // starting one is the transcript tab's job. // // Resolved over the timeline's assets, not the primary one: `hasTranscript` // below is already timeline-scoped (useCaptions), and mixing the two scopes // is what let a silent primary asset dead-end this button for a project whose // actual footage had speech. const gate = useTimelineTranscriptGate(); - const requestTimelineTranscripts = useTranscriptionStore((s) => s.requestTimelineTranscripts); + const transcriptions = useAssetTranscriptions(); + const transcriptionLabel = useTranscriptionLabel(); const isTranscribing = gate.state === "pending"; + // Timeline-scoped on purpose: the gate below answers for the timeline's + // assets, so the label must too — an off-timeline job must not relabel an + // enabled button. + const busyLabel = transcriptionBusyLabel( + firstTimelineBusyView(document, transcriptions) ?? + (isTranscribing ? { assetId: "", status: "running", phase: "loading-model" } : undefined), + transcriptionLabel, + ); const silentMedia = gate.state === "blocked" && gate.reason === "no-audio"; const engineError = gate.state === "blocked" && gate.reason === "failed" ? gate.message : null; @@ -180,14 +194,47 @@ export function CaptionsPane() { }; return ( -
-
-

{t("facets.captions")}

- - +
+
+ + +

{t("facets.captions")}

+ {onClose ? ( + + ) : null}
-
+
{t("captions.show")} ) : null} - + {/* No transcribe button here. This pane is reached from the transcript + tab, whose empty state carries the one gate — and two buttons for + one background pass is what made people believe captions were + transcribed separately from the transcript (issue #560). What is + worth saying here is whether a run is already going. */} + {isTranscribing ? ( +

+ + {busyLabel ?? t("captions.transcribing")} +

+ ) : null}
) : (

{/* The cue count is only meaningful while the layer is on — deriving cues short-circuits when it's off, so a "0 lines" reading there - would say the transcript is empty when it isn't. */} - {settings.enabled - ? t("captions.derivedFromTranscript", { count: cues.length }) - : t("captions.hiddenHint")} + would say the transcript is empty when it isn't. While a + regeneration is in flight the phase label matters more than the + count of cues about to be replaced. */} + {busyLabel ?? + (settings.enabled + ? t("captions.derivedFromTranscript", { count: cues.length }) + : t("captions.hiddenHint"))}

)} @@ -259,7 +319,7 @@ export function CaptionsPane() { padding: "12px 14px", border: "1px solid var(--border)", borderRadius: 10, - background: "var(--surface-warm)", + background: "var(--surface-2)", display: "flex", flexDirection: "column", gap: 8, @@ -310,7 +370,7 @@ export function CaptionsPane() { value={target} disabled={disabled || translating} onChange={(e) => setTarget(e.target.value)} - style={{ ...selectStyle, flex: 1 }} + style={{ ...selectStyle, flex: 1, minWidth: 0 }} > {TRANSLATION_LANGUAGES.map((language) => (
diff --git a/website/static/discord/index.html b/website/static/discord/index.html new file mode 100644 index 000000000..2cd2bbe1f --- /dev/null +++ b/website/static/discord/index.html @@ -0,0 +1,21 @@ + + + + + OpenScreen Discord + + + + + +

Redirecting to the OpenScreen Discord… If nothing happens, join here.

+ + + diff --git a/workbench/lib/oracles.ts b/workbench/lib/oracles.ts index d0435736d..e25f3350d 100644 --- a/workbench/lib/oracles.ts +++ b/workbench/lib/oracles.ts @@ -92,7 +92,10 @@ export function unplayableRegions(document: AxcutDocument): Array<{ kind: string document.timeline.clips, nextId, ); - const alive = new Set(projected.map((r) => r.id)); + // `underTrim` entries are emitted so a playhead parked on the cut can show what is + // underneath (issue #216) — they are precisely the regions playback never emits, so + // they stay DEAD here. Dropping them keeps this oracle's question unchanged. + const alive = new Set(projected.filter((r) => !r.underTrim).map((r) => r.id)); for (const region of family.regions) { // A zero-length span is stored and listed but can never play either. if (!alive.has(region.id) || region.endMs <= region.startMs) {