diff --git a/.github/workflows/ios-ipa.yml b/.github/workflows/ios-ipa.yml new file mode 100644 index 0000000..e505431 --- /dev/null +++ b/.github/workflows/ios-ipa.yml @@ -0,0 +1,117 @@ +name: Build OpenGlass iOS IPA + +on: + workflow_dispatch: + push: + branches: [codex/rokid-iphone-minicpm] + paths: ['ios/**', '.github/workflows/ios-ipa.yml'] + pull_request: + paths: ['ios/**', '.github/workflows/ios-ipa.yml'] + +permissions: + contents: read + +concurrency: + group: ios-ipa-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # Xcode 26.1.1 is listed on macos-15 and is the upstream-verified version: + # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md + runs-on: macos-15 + timeout-minutes: 90 + env: + DEVELOPER_DIR: /Applications/Xcode_26.1.1.app/Contents/Developer + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Check build tools + run: | + xcodebuild -version + cmake --version + python3 --version + - name: Test project preparation + run: python3 -m unittest discover -s ios/tests -p 'test_prepare*.py' -v + - name: Test Rokid wire protocol + run: bash ios/tests/run_protocol_tests.sh + - name: Test automatic voice conversation lifecycle + run: bash ios/tests/run_voice_loop_tests.sh + - name: Test Windows USB relay without devices + run: python3 -m unittest discover -s ios/windows -p 'test_usb_relay.py' -v + - name: Prepare pinned upstream and OpenGlass sources + run: | + python3 ios/scripts/prepare_upstream.py + plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo.xcodeproj/project.pbxproj + plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo/Info.plist + plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo/MiniCPM-V-demo.entitlements + - name: Restore pinned inference framework and Swift packages + id: restore_build_cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/llama.xcframework + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/.openglass-framework-key + tmp/ios-build/SourcePackages + key: ios-device-${{ runner.os }}-${{ runner.arch }}-xcode26.1.1-${{ hashFiles('ios/upstream-lock.json', 'ios/scripts/build_ipa.sh') }} + - name: Build unsigned device IPA + run: bash ios/scripts/build_ipa.sh + - name: Validate completed inference framework for caching + id: framework_cache_ready + if: ${{ !cancelled() && steps.restore_build_cache.outcome == 'success' && steps.restore_build_cache.outputs.cache-hit != 'true' }} + run: | + python3 - <<'PY' + import hashlib, os, plistlib, subprocess + from pathlib import Path + root = Path.cwd() + thirdparty = root / 'tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty' + framework = thirdparty / 'llama.xcframework' + stamp = thirdparty / '.openglass-framework-key' + valid = False + if stamp.is_file() and (framework / 'Info.plist').is_file(): + payload = (root / 'ios/upstream-lock.json').read_bytes() + payload += subprocess.check_output(['xcodebuild', '-version']) + payload += subprocess.check_output(['uname', '-m']) + expected = hashlib.sha256(payload).hexdigest() + if stamp.read_text().strip() == expected: + info = plistlib.loads((framework / 'Info.plist').read_bytes()) + for library in info.get('AvailableLibraries', []): + if (library.get('SupportedPlatform') == 'ios' + and not library.get('SupportedPlatformVariant') + and 'arm64' in library.get('SupportedArchitectures', [])): + binary = framework / library['LibraryIdentifier'] / library['LibraryPath'] / 'llama' + if binary.is_file() and binary.resolve().is_relative_to(framework.resolve()): + valid = subprocess.run(['lipo', '-verify_arch', 'arm64', str(binary)]).returncode == 0 + break + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f"valid={'true' if valid else 'false'}\n") + print('Completed inference framework cacheable:', valid) + PY + - name: Save completed framework even if the app build failed + if: ${{ !cancelled() && steps.framework_cache_ready.outputs.valid == 'true' && steps.restore_build_cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/llama.xcframework + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/.openglass-framework-key + tmp/ios-build/SourcePackages + key: ${{ steps.restore_build_cache.outputs.cache-primary-key }} + - name: Upload IPA for Windows signing + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: OpenGlass-iOS-unsigned-${{ github.run_number }} + path: | + artifacts/OpenGlass-1.0.ipa + artifacts/build-info.json + if-no-files-found: error + retention-days: 14 + compression-level: 0 + - name: Upload build logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: OpenGlass-iOS-build-logs-${{ github.run_number }} + path: tmp/ios-build/*.log + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index 593a3ff..498e69b 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,7 @@ live_sessions/ *.keystore *.jks *.apk +*.ipa +*.xcframework/ +*.dSYM/ +artifacts/ diff --git a/ios/README_zh.md b/ios/README_zh.md new file mode 100644 index 0000000..7b37e6a --- /dev/null +++ b/ios/README_zh.md @@ -0,0 +1,246 @@ +# Rokid → iPhone MiniCPM-V 4.6（Windows 开发） + +本目录把现有 Rokid 采集协议接入 iPhone，在手机本机运行 MiniCPM-V 4.6。Windows 可以编辑代码、编译眼镜 APK、下载 iOS 构建和协助安装；iOS 编译交给 GitHub Actions 的 macOS/Xcode runner，安装前在 Windows 用自己的 Apple 账号重新签名。不要求自有 MacBook。签名工具的常规入口、当前登录问题与本机认证测试版见第 3 节。 + +这是开发版本。源码、构建成功、签名安装成功、真实眼镜音视频到达、手机模型回答成功是不同的验证阶段；下方命令不会把“USB 已识别”当成端到端成功。 + +## 运行路径 + +无线版本使用 Rokid 采集端 **0.2.0**。眼镜加入 iPhone 个人热点（推荐），或与 iPhone 加入同一可互通的 Wi-Fi： + +```text +Rokid 摄像头 / 麦克风 → Wi-Fi → iPhone OpenGlass :18080 + → iPhone 离线 ASR → MiniCPM-V 4.6 → iPhone TTS + → 自动恢复听音，开始下一轮（每轮清空 KV cache） +``` + +这条路径不需要 Windows 转发；两端连接正常后可拔掉 USB。iPhone App 需要保持前台。首次配置热点 / Wi-Fi、权限和安装更新仍需操作设备。 + +双 USB 路径保留为回退方案，眼镜 Wi-Fi 可以保持关闭： + +```text +Rokid 摄像头 / 麦克风 + → 眼镜 OpenGlassRokidSensor 0.1.3 + → adb reverse（眼镜 USB） + → Windows 127.0.0.1:18080 + → pymobiledevice3 usbmux（iPhone USB） + → iPhone OpenGlass :18080 + → MiniCPM-V 4.6 本机视觉推理 → 系统语音播报 +``` + +Windows 在这个路径只转发数据，不运行视觉模型。拔掉任一 USB 线会中断此路径。确认双 USB 稳定后，可用同一 Wi-Fi 下的直接连接，去掉 Windows 数据中继： + +```text +Rokid → HTTP JPEG + WebSocket PCM → iPhone 局域网地址:18080 +``` + +眼镜本身需要支持 Android APK 和 Camera/Mic 权限；ADB 用于 Windows 安装与可选的首次配置，无线运行不需要 ADB。不能仅根据“Rokid”品牌判断所有型号兼容。0.2.0 延续 JPEG / PCM 协议，新增自动寻找手机和统一的断线重连；当前采样是 JPEG 约 1 fps、16 kHz 单声道 PCM16LE。模型按问题读取最新画面，不是对每个视频帧连续推理。 + +## 1. 准备 Windows 与手机 + +需要 Android Platform Tools、Python 3.11+、Git；从 Actions 下载产物的脚本还需要 GitHub CLI。Android 采集 APK 已安装时，无需重新编译；需要编译时沿用仓库的 `rokid_app/build_install.cmd`，另备 JDK 17 和 Android SDK。 + +1. 将眼镜和 iPhone 都用支持数据传输的 USB 线连接 Windows。 +2. iPhone 解锁，手动同意“信任此电脑”；眼镜确认 USB 调试授权。 +3. 按 [AltStore 官方 Windows 安装说明](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)安装 AltServer 及所要求的 Apple iTunes / iCloud 组件。官方默认流程使用 Apple 直接分发的版本；已安装 Store 版时，先参照官方故障排查，不要随意删除 Apple 驱动或服务。 +4. 在 iPhone 的“隐私与安全性”开启“开发者模式”，按系统要求重启并确认。签名配置的信任按系统提示在“通用 → VPN 与设备管理”中处理。 + +Apple 账号、密码及验证码只由本人输入所选签名工具的界面，不放入脚本、GitHub Secrets、日志或仓库。安装 iTunes / iCloud 组件与登录它们的云服务是两件事；本机认证测试版使用已安装的 Apple 组件，不要求分别登录 iCloud 和 iTunes。 + +在仓库根目录创建独立 Python 环境： + +```powershell +python -m venv .\ios\windows\.venv +.\ios\windows\.venv\Scripts\python.exe -m pip install -r .\ios\windows\requirements.txt +``` + +Python 命令未安装或打开 Microsoft Store 时，先安装真正的 Python，或者用其完整 `python.exe` 路径替换命令。 + +运行只读诊断： + +```powershell +.\ios\windows\diagnose.ps1 ` + -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" ` + -Python .\ios\windows\.venv\Scripts\python.exe +``` + +若有多个 Android 设备，在命令后加 `-Serial ''`。若同时安装 debug/release 两个采集 App，加 `-Package 'org.opensqz.openglass.rokid.debug'` 或实际已安装的 release 包。脚本会核验真实安装包、0.1.3 版本和 launcher Activity，不默认猜包名。分享诊断输出前删除设备序列号/UDID。 + +## 2. 获取 iOS IPA + +构建锁定 [MiniCPM-V-Apps](https://github.com/OpenBMB/MiniCPM-V-Apps) 与其 llama.cpp-omni 子模块的具体提交，见 `upstream-lock.json`；模型下载页沿用上游 MiniCPM-V 4.6 页面。iOS 工程由 `scripts/prepare_upstream.py` 在构建目录生成，不覆盖旧 Mac 上的项目。 + +在包含当前分支的 GitHub 仓库里打开 **Actions → Build OpenGlass iOS IPA**，选择该分支运行。工作流是 `.github/workflows/ios-ipa.yml`；首次编译原生推理框架会较慢。等待实际显示成功，再下载名为 `OpenGlass-iOS-unsigned-<运行编号>` 的 artifact，解压得到： + +- `OpenGlass-1.0.ipa`：未签名的真机 arm64 App。 +- `build-info.json`：构建来源信息。 + +也可在 Windows 用指定仓库和运行 ID 下载： + +```powershell +.\ios\windows\download-artifact.ps1 ` + -Repository '你的GitHub用户名/OpenGlass' ` + -RunId 123456789 +``` + +`RunId` 是 Actions 运行 URL 的数值 ID，不是 artifact 后缀的运行编号；示例值必须替换。脚本核验工作流、成功状态和未过期产物，再检查 `build-info.json` 的源提交与该运行一致、`signed=false`、bundle ID正确，以及 IPA中存在且可读取 `Payload/OpenGlass.app/Info.plist`。它不把IPA中的任意路径解压到磁盘，并输出源提交与 IPA SHA-256；目标目录存在时退出，避免覆盖旧产物。GitHub CLI 未登录时，由本人先运行 `gh auth login`。 + +## 3. 用免费 Apple 账号签名安装 + +CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobiledevice3 apps install` 和 USB 端口转发都不会把 unsigned IPA 变成可运行的签名 App。 + +**常规入口：从 Windows AltServer 直接侧载。** 本次现场的 AltServer 1.8 在验证码阶段出现 `-22411`，因此以下入口尚未在本机成功完成安装；不要把它视为已验证的解决办法。手机已经有旧 OpenGlass 和自己编译的 MiniCPM 时，先装 AltStore 可能占用第三个免费侧载名额，使新 OpenGlass 成为第四个而无法安装。直接侧载无需增加 AltStore 这个手机 App： + +1. 保持 iPhone 通过 USB 连接、解锁并已信任电脑；确认 Apple Mobile Device Service 可用。Windows能看到iPhone图标或iTunes已安装，不等于该服务已准备好。 +2. 启动 Windows 的 **AltServer**，找到任务栏右下角托盘里的图标。 +3. **按住 Shift，再点击 AltServer 托盘图标**，选择 **Sideload .ipa…**，并选择目标 iPhone（界面有设备子菜单时）。 +4. 在 Windows 文件选择框选中构建下载得到的 **OpenGlass-1.0.ipa**。 +5. 本人在 AltServer 界面输入自己的 Apple 账号并完成验证，等待签名和安装结果；随后在 iPhone 按提示信任签名配置、确认开发者模式，再打开新 App。 + +这条 Windows 直接侧载入口及其无需预装 AltStore 的行为见 [AltServer 官方发布说明的 Direct .ipa Sideloading](https://faq.altstore.io/release-notes/altserver)。保留下载的 IPA；免费签名需要每 7 天用同一账号、同一 IPA 的上述步骤重新签名安装。此路径没有 AltStore 的自动刷新，不需要先把 IPA 转到手机“文件”App。 + +源 bundle ID 为 `org.opensqz.openglass.ios`；重签工具可能附加账号相关标识。显示名称是 **OpenGlass**。旧手机 App 若由另一账号/Team 签名，不能保证覆盖安装、共享模型文件或保留原数据；先保留旧 App，核对错误提示与签名来源，避免通过卸载来试错。 + +免费账号的活跃侧载 App 上限为 3 个，AltStore 本身也占名额；直接侧载只是省去 AltStore 的名额，并不会提高上限。若仍提示超限，先核对当前签名/活跃 App 清单，**不要卸载旧 OpenGlass 或 MiniCPM 来试错**。官方限制见 [AltStore Classic 开发者说明](https://developer.altstore.io/)及[错误码](https://faq.altstore.io/altstore-classic/error-codes)。如果原 OpenGlass“点不开”，签名过期只是可能原因之一，还应区分未信任、开发者模式、闪退和模型内存问题。 + +**后续可选：用 AltStore 管理自动刷新。** 确认有空余名额后，按 [官方安装流程](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)从 AltServer 安装 AltStore Classic，再把 IPA 保存到 iPhone“文件”App，在 AltStore **My Apps → +** 中安装并管理。直接用 AltServer 安装的 App 不能假定会自动出现在 AltStore 的管理列表；要以 AltStore 里实际显示的 App 和到期时间为准。自动刷新仍依赖可达的 AltServer及系统后台条件，临近到期应主动检查。 + +**本次本地交付：OpenGlass Local Signer 测试版。** Sideloadly 0.70.1 使用 Local Anisette 后可以输入验证码，但本机仍收到 GSA HTTP 503。这个错误本身不能证明密码错误或 TLS 被拦截。为继续只使用本地认证组件，准备了基于 [iloader v2.3.9](https://github.com/nab138/iloader/tree/aa9aa3779e15498180e0205158b7ff91536877c6) 和 isideload 0.4.6 的修改版，保存于本机 `%USERPROFILE%\Downloads\OpenGlass-Local-Signer`。它不是官方 iloader 发布；目录内源码包及 `BUILD_WINDOWS.md` 提供构建输入和说明。 + +主程序旁必须保留 `local-apple-identity.exe` 与 `local-coreadi-helper.exe`。它们核验已安装 Apple DLL 的固定 SHA256 和签名，通过本机管道取得原生身份及认证数据；没有远程 Anisette 备用服务，不自动擦除或重新注册原生身份。当前构建锁定本机 Apple 组件版本，不保证适用于其他版本的电脑。登录、验证码验证、申请签名证书和描述文件仍需要联网到 Apple；“本地认证”不等于离线签名。 + +1. 运行 `OpenGlass-Local-Signer.exe`，本人在 Apple ID 区域登录并填写验证码；保存密码默认不勾选。 +2. 选择 USB 连接的 iPhone，点击 **Import IPA**，选择本次构建的 IPA；选中文件后即开始签名安装。 +3. 若提示开发证书已满，先取消并确认哪些旧 App 依赖已有证书；程序不会默认选择全部撤销。 +4. 用手机安装列表核对新 App 的实际 bundle ID、版本和文件共享状态，再导入模型。仅有“账号登录成功”或 Windows 程序已打开不代表 iPhone 安装完成。 + +测试版已通过 6 项管道协议测试、前后端构建、本机组件调用，以及携带公共客户端描述的 Apple URL-bag GET / TLS 校验。2026-10-16，本人操作后 Apple 账号登录和签名成功；首次安装因已有 3 个免费开发 App 被手机拒绝。用户明确选择卸载旧 OpenGlass 后，复用已签名文件安装新版成功，USB 查询确认版本 `1.0`、build `3`、文件共享可用，MiniCPM 示例与 CyanGlass 均保留。签名工具会给新 bundle ID 追加 Team ID；只改 IPA 的原始 bundle ID 不能假定能覆盖旧 Xcode App。自动更新已关闭，避免被上游的不同认证配置替换。 + +## 4. 在 iPhone 准备模型 + +1. 打开新 **OpenGlass**，进入右上角“模型下载”。 +2. 在 MiniCPM-V 4.6 下载页完成 **LLM 与 VPM 两个 GGUF 文件**。这个 App 有独立沙盒；其他 TestFlight/App 已下载的模型不会自动出现在这里。下载与首次资源准备需要网络。 +3. 模型已准备好时，打开 App 会自动启动接收并加载模型。首次刚从下载页返回时点一次“启动接收”。初次验证可先关闭“Rokid 离线语音提问”，用文字问题排除语音权限影响。 +4. 保持 OpenGlass 在前台。进入后台会挂起接收和语音；返回后自动恢复此前运行状态。主动点“停止接收”或“暂停语音”后不会自动重启。本实现不保证锁屏长期运行。 + +模型文件不能混用 MiniCPM-o 4.5、其他尺寸或未经验证的 GGUF 转换。加载会检查文件存在、GGUF 头和锁定上游对应的 MD5；首次完整校验在后台进行，后续复用文件大小/修改时间匹配的校验记录。文件名与预期校验值由锁定上游的 MiniCPM-V 4.6 常量决定；模型通过校验仍需真机推理验证。 + +## 5A. 双 USB 接入（先验证这条） + +确认 iPhone 已启动接收后，在 Windows 仓库根运行： + +```powershell +.\ios\windows\start-usb-relay.ps1 ` + -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" ` + -Python .\ios\windows\.venv\Scripts\python.exe +``` + +Windows PowerShell 若提示“在此系统上禁止运行脚本 / PSSecurityException”，可仅为这次启动允许本地脚本，使用下面的命令；把路径替换为实际仓库绝对路径。`-ExecutionPolicy RemoteSigned` 只影响新进程，不永久修改用户或系统设置，也不会覆盖组织的组策略： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGlass\ios\windows\start-usb-relay.ps1' +``` + +本项目生成或通过 Git 检出的本地脚本可以使用这条命令。若仍报执行策略错误，先检查 `Get-ExecutionPolicy -List` 和文件的来源标记，不全局降低策略。作用范围见 [Microsoft 官方执行策略说明](https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_execution_policies?view=powershell-5.1)。 + +仅连接一副授权眼镜和一台 iPhone 时自动选择；多设备时显式加 `-Serial '<眼镜USB序列号>'`、`-IPhoneUdid ''`。用下列只读命令查询 iPhone USB UDID： + +```powershell +.\ios\windows\.venv\Scripts\python.exe -m pymobiledevice3 usbmux list --usb --simple +``` + +脚本先检查既有 ADB reverse 映射，随后在 Windows `127.0.0.1` 绑定端口，通过 iPhone USB 查询 `/health` 并核验 `service: openglass-ios`；通过后才建立独占反向映射并启动眼镜采集 App。App 无需 Wi-Fi 地址，实际保存的采集目标是眼镜自己的 `http://127.0.0.1:18080`。 + +Windows 原 Python 面板占用了 18080 时，先在该面板正常停止；也可以为本次中继换 Windows 端口，不改变 iPhone 监听端口： + +```powershell +.\ios\windows\start-usb-relay.ps1 ` + -Python .\ios\windows\.venv\Scripts\python.exe -LocalPort 18081 +``` + +如果眼镜18080已有其他会话的 reverse 映射，脚本会退出，不抢占。可结束原会话，或显式加 `-RokidPort 18081`；脚本会同步传入正确采集 URL。不要同时启动旧电脑面板去重配同一眼镜。 + +运行期间另开终端查看手机接收计数（修改了 `LocalPort` 就使用对应端口）： + +```powershell +Invoke-RestMethod http://127.0.0.1:18080/health +``` + +`image_count`、`audio_packets_in` 应随采集增长。主终端按 **Ctrl+C** 正常停止，会关闭本次转发器，并移除本次创建、且未被其他会话改变的 reverse 映射。不会杀死所有 Python/ADB 进程，不会卸载 App。眼镜采集 App 可能继续重连，必要时在眼镜界面手动停止。 + +底层使用官方 [pymobiledevice3 v11.32.0 TCP forwarder](https://github.com/doronz88/pymobiledevice3/blob/v11.32.0/pymobiledevice3/tcp_forwarder.py)，限定 USB。其 CLI 对应 `usbmux forward `，这里的包装脚本另外处理设备选择、健康检查和清理。普通 App TCP 转发不使用 iOS 开发调试隧道，不要求将 Apple 配对文件上传到任何服务。 + +## 5B. Wi-Fi 直连（推荐 iPhone 个人热点） + +先安装新 iOS IPA 和 Rokid 0.2.0 APK。iOS 使用同一 Apple 账号、同一 bundle ID 覆盖安装，保留现有模型；无需卸载或重新下载模型。本机旧眼镜 APK 的原签名文件不可用，因此交付并存包 **OpenGlass WiFi**（`org.opensqz.openglass.rokid.wifi`，`0.2.0-wifi`），保留旧采集 App。使用新版前先在旧 App 中停止采集，避免两个程序争用相机和麦克风；新 App 首次打开需允许相机和麦克风。 + +1. iPhone“设置 → 个人热点 → 允许其他人加入”。眼镜找不到热点时可开启“最大兼容性”。首次连接时停留在热点页面，直到眼镜已加入。具体开关见 [Apple 个人热点说明](https://support.apple.com/zh-cn/111785)。热点依赖运营商支持；它分享的是蜂窝网络，不是酒店 Wi-Fi 的中继。 +2. 通过 Rokid 官方 App / 设备的 Wi-Fi 设置，让眼镜加入这个热点；密码仅在设备设置内输入。OpenGlass 不修改 Wi-Fi 密码和系统网络。 +3. iPhone 打开 OpenGlass，允许“本地网络”和语音识别；已有模型时自动启动接收，等待模型就绪即可。页面显示固定的 `OpenGlass-xxxxxxxx` 接收名称。请保持“Rokid 离线语音提问”和“朗读回答”开启。 +4. 启动眼镜 **OpenGlass WiFi**，使用“自动连接”。眼镜寻找 `_openglass._tcp` 服务，并探测当前 Wi-Fi 网关的 `:18080/health`；确认是 OpenGlass iOS 接收端后才发送音视频。地址变化后会重新发现，不把热点地址写死。等手机显示“正在听 Rokid 麦克风（离线识别）”后开始说话。 +5. 确认手机出现新画面、自动语音状态后，拔掉两根 USB，等待十秒并连续问三个问题。回答播完后再说下一个问题，不需要点“结束语句”或“用最新画面提问”。 + +眼镜暂留 USB 时，可在 Windows 手动切到自动发现模式；这条命令只做一次配置与启动，不是数据中继： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGlass\ios\windows\configure-rokid.ps1' -AutoDiscover -Package org.opensqz.openglass.rokid.wifi +``` + +普通 Wi-Fi / 酒店 Wi-Fi 也可用：将两台设备加入同一网络。网络必须允许设备互访；若酒店启用了客户端隔离，或眼镜无法完成网页认证，应改用个人热点。Bonjour 被阻止但设备仍能直连时，可从 iPhone 页面取 IPv4 地址手动指定： + +```powershell +.\ios\windows\configure-rokid.ps1 ` + -IPhoneAddress '192.168.1.42' ` + -Package org.opensqz.openglass.rokid.wifi ` + -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" +``` + +替换示例 IP。手动地址模式用于明确指定接收设备，换网后可重新选“自动连接”。脚本不打开/修改眼镜 Wi-Fi。若 Windows 未加入手机所在网络而眼镜已加入，Windows 的接收端检查可能失败；可使用自动发现，或确认原因后加 `-SkipReceiverCheck`。 + +无线连通性须在这副眼镜和这台 iPhone 上实测，不能把“已编译”当成“热点已跑通”。检查时必须拔掉 USB，并确认新画面仍在更新，防止误把原 USB 链路当成无线成功。 + +## 6. 验证对话和音频 + +先确认预览方向正确；主界面可选0/90/180/270°旋转，默认270°沿用原电脑路径。输入“我面前有什么？”后点“用最新画面提问”，验证手机能读到当前眼镜画面并生成回答。 + +然后启用“Rokid 离线语音提问”，选择中文或英文，允许系统语音识别权限。输入仍来自眼镜的 PCM 流，App 不打开 iPhone 麦克风。Apple 设备端识别资源可用时执行离线识别；不可用会显示错误并保留文字提问，不改走云端。说完停顿后，识别文本与当前图片自动送给 MiniCPM-V 4.6。回答由系统 TTS 朗读，结束后自动恢复听音；每轮推理前清空 KV cache，不保留此前问题或图片。可在 iPhone 控制中心选择已配对的眼镜作为声音输出，前提是该型号支持相应蓝牙音频功能。 + +自动循环为“听问题 → 看图推理 → 朗读回答 → 等待回声消退 → 继续听问题”。推理/播报时丢弃输入语音，不积压旧问题；空白识别、短暂识别错误、音频重连与新画面恢复均由状态机处理。权限拒绝、离线语言资源缺失、模型错误仍会明确显示，主动暂停必须由用户恢复。这是逐轮交互，暂不支持边播报边打断。 + +正常空白识别不增加错误退避：有收尾缓冲就立即进入下一轮，否则等下一包声音到达再开始；真实识别错误仍按0.5～8秒退避。仅在 `endAudio` 到空结果之间暂存最多2.2秒的新声音，空结果后须在0.5秒内交给下一轮，避免收尾期间漏掉下一句。已送入识别器的声音不重放；非空结果、识别错误、暂停、断线和后台切换均清空缓冲。没有文字时的短促噪声不再触发1.3秒收尾，安静时按25秒轮换识别任务。 + +手机“自动语音”状态显示实际听音、收尾、等待或播报状态及已完成轮数。`/health` 的 `conversation_phase` 显示 `waiting/listening/finishing/cooldown/thinking/speaking/paused`，`completed_turns` 记录本次接收会话中完成的回答数，`app_build` 用于核对安装版本。`asr` 提供实际送入识别器的样本数、音量、非空结果数和错误域/错误码，每秒刷新；不返回识别文本、音频或回答正文。若音频包持续增长但 `asr.accepted_samples` 不增长，先检查 `asr.gate`；若样本和音量增长但没有非空结果，再检查识别错误。测试至少三轮，并留空静音约30秒后再提问，确认没有空问题、重复回答或需要手动唤醒。 + +## 排障 + +| 现象 | 先检查 | +| --- | --- | +| Windows看得到iPhone，usbmux列表为空/报错 | 数据线、手机解锁与信任、Apple Mobile Device Service；PnP识别不等于配对完成 | +| USB中继提示手机 `/health` 不可达 | 新App是否成功签名安装并已启动接收；是否进入后台；模型加载失败是否导致接收停止 | +| 端口或ADB reverse已存在 | 在原会话中正常停止，或指定其他本机/眼镜端口；脚本不会杀进程抢占 | +| `/health` 成功但计数不增长 | 眼镜相机/麦克风权限、采集App版本、ADB授权、所选设备；Wi-Fi模式另查网络互通 | +| 无线模式没有画面，眼镜显示 Wi-Fi 已开启 | 已开启不等于已连接。先确认眼镜已加入热点，再切回手机 OpenGlass；切到热点设置期间，接收服务会因进入后台而暂停。两端恢复后自动重连，无需重跑电脑脚本 | +| 画面到达但模型未就绪 | 两个正确的4.6模型是否下载完成、磁盘空间、模型加载报错；其他App模型沙盒不共享 | +| 有文本回答但没声音 | “朗读回答”开关、iPhone音量/音频路由、蓝牙配对；看界面实际输出设备 | +| 语音不可用但文字正常 | 系统语音识别权限与该语言的设备端资源；该功能不向云识别降级 | +| 切后台后断流 | 预期行为；回前台自动恢复此前运行意图；主动停止不会自动恢复 | +| 免费签名几天后点不开 | 直接侧载方式每7天用AltServer重签；AltStore管理方式查看剩余天数并刷新。不要先删除App及模型数据 | + +## 验证记录 + +- 本地脚本检查：PowerShell语法解析通过；USB中继使用伪设备测试了端口占用、已有reverse、并发映射冲突、接收端不可达、成功启动与只清理自身映射等情况。 +- [CI运行38890873970](https://github.com/OpenSQZ/OpenGlass/actions/runs/38890873970)成功，源提交为 `c11c87b39b5798f8b4218d3a47d5b6cc68e56091`：1399项wire protocol检查、7项工程准备测试、7项USB中继测试通过，并生成未签名的 `OpenGlass-1.0.ipa`（artifact `OpenGlass-iOS-unsigned-3`）。IPA SHA256 为 `DA76A28A87195C43EA80ECF11366724180480B5D6502DEA27A4D1AF800AD00DB`。 +- 现场设备识别为 `iPhone18,2`、iOS `26.6.2`。新版 `1.0` / build `3` 已通过本机认证签名并安装；两个对应 4.6 模型已校验并导入。2026-10-18 用户反馈：USB 连接下眼镜图像、声音和 iPhone MiniCPM 推理已跑通，仍需手动进行单轮交互。本次无线与自动多轮改动另行验证，不能继承旧版的端到端结论。 +- 无线/自动语音 iOS 源提交 `087766314c0d817a78b65c6e6a93d46b281d5576` 通过 [CI 39057876255](https://github.com/OpenSQZ/OpenGlass/actions/runs/39057876255)，生成 build `5`。1399 项协议检查、2087 项语音循环断言、7 项工程准备测试、7 项 USB relay 测试通过。IPA SHA256：`D5A8F689048DFE09E1C4F4AF4EE67E770E5FE01BE3883A0A4B414804B39BFCB7`。Windows 免费签名更新后，USB 查询已确认手机安装 build `5`，两份模型文件仍完整保留。 +- Rokid `0.2.0-wifi` / code `5` 的 `assembleWifi`、7 项 `testWifiUnitTest` 与原 `assembleDebug` 通过；APK SHA256：`A5A64849FEAC796D4688B60833E647FA3DACE1CB3F5A16BD8935AA0EA5623A13`。并存包 `org.opensqz.openglass.rokid.wifi` 已安装，旧 debug 采集 App 保留；新 App 的相机和麦克风权限已确认允许。 +- 2026-10-18 热点真机测试：眼镜直接向当前 iPhone 热点网关发送图像和音频，手机接收计数持续增长。拔掉眼镜 USB 后，ADB 已无设备，手机仍持续收流，`completed_turns` 增至 `3`。随后用户确认：无需点按钮即可连续问答，且眼镜和 iPhone 两根 USB 均拔掉后仍正常。已验证本机 ASR → MiniCPM-V 4.6 → TTS 的无线自动多轮流程；酒店 Wi-Fi 尚未实测。 +- 包含 Android 最终改动的提交 `c439fa3` 也通过 [CI 39058773419](https://github.com/OpenSQZ/OpenGlass/actions/runs/39058773419)。其 iOS Swift 源码与已安装的 build `5` 相同，无需再次安装。 +- 后续用户确认单行眼镜界面和首轮 ASR/TTS 正常，但再次提问出现空识别，说明此前成功记录不足以证明持续对话可靠。此次修复空轮退避和收尾期间丢音，并增加 ASR 操作状态诊断；新版端到端多轮结果待安装后复测。 +- 语音修复提交 `57589c0` 通过 [CI 39061505661](https://github.com/OpenSQZ/OpenGlass/actions/runs/39061505661)：2176项语音循环/诊断/交接断言、1399项协议检查、7项工程准备测试、7项USB中继测试及真机Release编译通过。未签名IPA build `11` 已下载，manifest、ZIP完整性、Info.plist版本及bundle ID已核验；SHA256为 `1B9839531088356F914B6510C33377A971CFBF6EF162915106DC196413305B6A`。Rokid code `8` 的单行界面更新已覆盖安装并确认相机/麦克风授权；APK SHA256为 `A06D29A45CC2D346A331EFB7AA63B8FA3220963EEC3AABE83C57554A4CFD80F5`。 +- 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 + +可重复执行无设备测试： + +```powershell +python -m unittest discover -s .\ios\windows -p test_usb_relay.py -v +``` diff --git a/ios/Sources/OpenGlassModelSession.swift b/ios/Sources/OpenGlassModelSession.swift new file mode 100644 index 0000000..42d382d --- /dev/null +++ b/ios/Sources/OpenGlassModelSession.swift @@ -0,0 +1,197 @@ +import Foundation +import Combine + +/// Owns exactly one native model context. All entry points run on the main actor; +/// a busy session rejects new work instead of queueing frames or questions. +@MainActor +final class OpenGlassModelSession { + enum Phase: Equatable { + case unloaded, loading, ready, preparing, generating, stopping, failed + } + + enum SessionError: LocalizedError { + case busy, missingFiles, cancelled, restartRequired, invalidFile(String) + var errorDescription: String? { + switch self { + case .busy: return "上一轮尚未结束，请稍候。" + case .missingFiles: return "请先在“模型下载”中下载 MiniCPM-V 4.6 的 LLM 和 VPM 文件。" + case .cancelled: return "已停止本轮。" + case .restartRequired: return "底层推理超时，已锁定模型以保护内存。请关闭并重新打开 App。" + case .invalidFile(let name): return "模型文件无效：\(name)。请在模型下载页重新下载。" + } + } + } + + var onChange: ((Phase, String) -> Void)? + var onOutput: ((String) -> Void)? + private(set) var phase: Phase = .unloaded + private(set) var isBusy = false + private(set) var isLoaded = false + private(set) var requiresAppRestart = false + private var wrapper: MTMDWrapper? + private var tokenSubscription: AnyCancellable? + private var discardResult = false + private var output = "" + // A timed-out worker cannot be joined through the upstream API. Keep its + // context alive even if UIKit later discards the view controller. + private static var quarantinedContexts: [MTMDWrapper] = [] + + static var modelFiles: [String] { + [MiniCPMModelConst.modelv46_FileName, MiniCPMModelConst.mmprojv46_FileName] + } + + func load() async throws { + guard !requiresAppRestart else { throw SessionError.restartRequired } + guard !isBusy else { throw SessionError.busy } + if isLoaded { return } + isBusy = true + discardResult = false + transition(.loading, "正在加载 MiniCPM-V 4.6…") + defer { isBusy = false } + do { + let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] + let files = Self.modelFiles.map { documents.appendingPathComponent($0) } + for file in files { + guard FileManager.default.fileExists(atPath: file.path) else { throw SessionError.missingFiles } + // Reject obvious truncation before doing full artifact validation. + let handle = try FileHandle(forReadingFrom: file) + let header = try handle.read(upToCount: 4) + try handle.close() + guard header == Data("GGUF".utf8) else { throw SessionError.invalidFile(file.lastPathComponent) } + } + // The upstream download UI reports completion before its background + // checksum finishes. A GGUF header/filename alone cannot prove that + // these are the matching 4.6 LLM and projector. Reuse its cached + // validation receipts, and keep first-time hashing off the UI actor. + let checksums = [MiniCPMModelConst.modelv46_Q4_K_M_MD5, + MiniCPMModelConst.modelMMProjv46_MD5] + transition(.loading, "正在校验 MiniCPM-V 4.6 模型文件…") + let invalidFile = await Task.detached(priority: .userInitiated) { + for (file, checksum) in zip(files, checksums) { + if !MBModelArtifactStore.validate(file, expectedMD5: checksum) { + return Optional(file.lastPathComponent) + } + } + return Optional.none + }.value + if let invalidFile { throw SessionError.invalidFile(invalidFile) } + try checkStopped() + transition(.loading, "正在加载 MiniCPM-V 4.6…") + let engine = MTMDWrapper() + wrapper = engine + engine.setEnableThinking(false) + let params = MTMDParams(modelPath: files[0].path, mmprojPath: files[1].path, + nPredict: 160, nCtx: 4096, nThreads: 4, + temperature: 0.2, useGPU: true, mmprojUseGPU: true, + warmup: false, nUbatch: 512, imageMaxSliceNums: 1) + try await engine.initialize(with: params) + engine.setModelVersion(46) + engine.decodeTokenLimit = 160 + tokenSubscription = engine.$currentToken.dropFirst().sink { [weak self] token in + guard let self, self.phase == .generating || self.phase == .stopping else { return } + self.output += token.content + if !self.discardResult { self.onOutput?(self.output) } + } + isLoaded = true + transition(.ready, discardResult ? "模型已加载；接收已停止。" : "MiniCPM-V 4.6 已就绪") + } catch { + // initialize has returned; no native worker is left running on this path. + if let engine = wrapper { await engine.reset() } + wrapper = nil + isLoaded = false + transition(.failed, error.localizedDescription) + throw error + } + } + + func ask(question: String, imageURL: URL?, answerLanguage: String = "zh-CN") async throws -> String { + guard !requiresAppRestart else { throw SessionError.restartRequired } + guard !isBusy else { throw SessionError.busy } + guard isLoaded, let engine = wrapper else { throw SessionError.missingFiles } + isBusy = true + discardResult = false + output = "" + engine.decodeTokenLimit = 160 + transition(.preparing, imageURL == nil ? "正在处理问题…" : "正在理解最新画面…") + defer { isBusy = false } + do { + // Previous turn has fully completed before clearing KV. Never call + // upstream stopGeneration(), which publishes completion too early. + engine.clearKVCacheForNewTurn() + if let imageURL { + try await engine.addImageInBackground(imageURL.path, timeoutSeconds: 180) + } + try checkStopped() + let instruction = answerLanguage.hasPrefix("en") + ? "Answer briefly in English, suitable for reading aloud, in at most 80 words. Describe only what you can confirm.\n" + : "请用简洁中文回答，适合语音朗读，不超过100字。只描述你确实能确认的内容。\n" + let prompt = instruction + question + try await engine.addTextInBackground(prompt) + try checkStopped() + transition(.generating, "正在生成回答…") + try await engine.startGeneration() + // startGeneration launches an internal task, so await its terminal + // state before letting another request touch the context. + while engine.generationState == .generating { + try? await Task.sleep(nanoseconds: 30_000_000) + } + if case .failed(let error) = engine.generationState { throw error } + try checkStopped() + let answer = output.trimmingCharacters(in: .whitespacesAndNewlines) + transition(.ready, "回答完成") + return answer + } catch { + if let mtmdError = error as? MTMDError, case .timeout = mtmdError { + // The upstream watchdog can return while C++ still uses ctx. + // Retain the wrapper, never reset/free or start a second call. + requiresAppRestart = true + Self.quarantinedContexts.append(engine) + transition(.failed, SessionError.restartRequired.localizedDescription) + throw SessionError.restartRequired + } + if let sessionError = error as? SessionError, case .cancelled = sessionError { + transition(.ready, "本轮已结束，可重新启动。") + } else { + // The native operation has returned. Reinitializing next time + // is safer than reusing a context after a prefill error. + await engine.reset() + tokenSubscription = nil + wrapper = nil + isLoaded = false + transition(.failed, error.localizedDescription) + } + throw error + } + } + + func requestStop() { + discardResult = true + guard isBusy else { return } + if phase == .generating { + // A Swift-only flag read after mb_mtmd_loop returns. The next token + // finishes normally, including native perf bookkeeping. + wrapper?.decodeTokenLimit = 1 + } + transition(.stopping, "正在结束本轮推理…") + } + + /// Used before entering the download UI, which can replace model files. + func unload() async throws { + guard !requiresAppRestart else { throw SessionError.restartRequired } + guard !isBusy else { throw SessionError.busy } + if let engine = wrapper { await engine.reset() } + wrapper = nil + tokenSubscription = nil + isLoaded = false + transition(.unloaded, "模型未加载") + } + + private func checkStopped() throws { + if discardResult { throw SessionError.cancelled } + } + + private func transition(_ phase: Phase, _ message: String) { + self.phase = phase + onChange?(phase, message) + } +} diff --git a/ios/Sources/OpenGlassSpeechDiagnostics.swift b/ios/Sources/OpenGlassSpeechDiagnostics.swift new file mode 100644 index 0000000..322fe76 --- /dev/null +++ b/ios/Sources/OpenGlassSpeechDiagnostics.swift @@ -0,0 +1,77 @@ +/// Operational metadata only: never retains speech, audio or error descriptions. +struct OpenGlassSpeechDiagnostics { + enum State: String { + case idle, listening, finishing + } + + private(set) var state: State = .idle + private(set) var sessions = 0 + private(set) var acceptedSamples = 0 + private(set) var sessionSamples = 0 + private(set) var lastRMS = 0.0 + private(set) var sessionPeakRMS = 0.0 + private(set) var partialCount = 0 + private(set) var nonemptyPartialCount = 0 + private(set) var emptyFinalCount = 0 + private(set) var nonemptyFinalCount = 0 + private(set) var failureCount = 0 + private(set) var lastFailureDomain = "" + private(set) var lastFailureCode: Int? + private(set) var lastEndReason = "not_started" + + mutating func reset() { + self = Self() + } + + mutating func started() { + state = .listening + sessions += 1 + sessionSamples = 0 + lastRMS = 0 + sessionPeakRMS = 0 + lastEndReason = "none" + } + + mutating func appended(samples: Int, rms: Double) { + guard state == .listening, samples > 0 else { return } + acceptedSamples += samples + sessionSamples += samples + // Invalid measurements must not introduce NaN/Infinity into diagnostics. + lastRMS = rms.isFinite ? max(0, rms) : 0 + sessionPeakRMS = max(sessionPeakRMS, lastRMS) + } + + mutating func partial(nonempty: Bool) { + guard state != .idle else { return } + partialCount += 1 + if nonempty { nonemptyPartialCount += 1 } + } + + mutating func finishing(reason: String) { + guard state == .listening else { return } + state = .finishing + lastEndReason = reason + } + + mutating func completed(nonempty: Bool, reason: String) { + guard state != .idle else { return } + if nonempty { nonemptyFinalCount += 1 } + else { emptyFinalCount += 1 } + state = .idle + lastEndReason = reason + } + + mutating func failed(domain: String, code: Int, reason: String) { + // A recognizer can fail synchronously before a session starts. + failureCount += 1 + state = .idle + lastFailureDomain = domain + lastFailureCode = code + lastEndReason = reason + } + + mutating func stopped(reason: String) { + state = .idle + lastEndReason = reason + } +} diff --git a/ios/Sources/OpenGlassSpeechHandoffBuffer.swift b/ios/Sources/OpenGlassSpeechHandoffBuffer.swift new file mode 100644 index 0000000..8c3aaeb --- /dev/null +++ b/ios/Sources/OpenGlassSpeechHandoffBuffer.swift @@ -0,0 +1,57 @@ +import Foundation + +/// In-memory PCM received only after endAudio, for one empty-result handoff. +/// Audio already appended to a recognition request must never enter this buffer. +struct OpenGlassSpeechHandoffBuffer { + enum Phase: Equatable { + case idle, collecting, ready + } + + private(set) var phase: Phase = .idle + private let capacity: Int + private var bytes = Data() + private var readyAt: TimeInterval? + + var byteCount: Int { bytes.count } + + func hasReadyAudio(at now: TimeInterval) -> Bool { + guard phase == .ready, !bytes.isEmpty, let readyAt else { return false } + return now >= readyAt && now - readyAt <= 0.5 + } + + init(capacity: Int = 70_400) { + let nonnegativeCapacity = max(0, capacity) + self.capacity = nonnegativeCapacity - nonnegativeCapacity % 2 + } + + mutating func begin() { + clear() + phase = .collecting + } + + mutating func append(_ data: Data) { + guard phase == .collecting, bytes.count < capacity else { return } + // Preserve the start of an utterance instead of replacing it with its tail. + bytes.append(contentsOf: data.prefix(capacity - bytes.count)) + } + + mutating func complete(empty: Bool, now: TimeInterval) { + guard empty, phase == .collecting else { clear(); return } + phase = .ready + readyAt = now + } + + mutating func takeReadyAudio(now: TimeInterval) -> Data { + guard phase == .ready else { return Data() } + guard hasReadyAudio(at: now) else { clear(); return Data() } + let audio = bytes + clear() + return audio + } + + mutating func clear() { + phase = .idle + bytes.removeAll(keepingCapacity: false) + readyAt = nil + } +} diff --git a/ios/Sources/OpenGlassSpeechSession.swift b/ios/Sources/OpenGlassSpeechSession.swift new file mode 100644 index 0000000..ffbaf24 --- /dev/null +++ b/ios/Sources/OpenGlassSpeechSession.swift @@ -0,0 +1,305 @@ +import Foundation +import Speech +import AVFoundation + +/// Recognizes only incoming Rokid PCM16LE, 16 kHz, mono. It never opens the +/// iPhone microphone and never falls back to Apple's cloud recognition. +@MainActor +final class OpenGlassSpeechSession { + var onPartial: ((String) -> Void)? + var onFinal: ((String) -> Void)? + var onError: ((Failure) -> Void)? + var onStatus: ((String) -> Void)? + private(set) var isListening = false + private(set) var diagnostics = OpenGlassSpeechDiagnostics() + var isActive: Bool { request != nil } + var isFinishing: Bool { ending } + var hasPendingHandoffAudio: Bool { + handoff.hasReadyAudio(at: ProcessInfo.processInfo.systemUptime) + } + private var recognizer: SFSpeechRecognizer? + private var request: SFSpeechAudioBufferRecognitionRequest? + private var task: SFSpeechRecognitionTask? + private var timer: Timer? + private var finishTimer: Timer? + private var sessionID = UUID() + private var latestText = "" + private var startedAt = ProcessInfo.processInfo.systemUptime + private var lastVoiceAt: TimeInterval? + private var lastTranscriptAt: TimeInterval? + private var lastPacketAt = ProcessInfo.processInfo.systemUptime + private var pendingByte: UInt8? + private var handoff = OpenGlassSpeechHandoffBuffer() + private var ending = false + private let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, + sampleRate: 16_000, channels: 1, interleaved: false)! + + static func authorize() async -> Bool { + if SFSpeechRecognizer.authorizationStatus() == .authorized { return true } + return await withCheckedContinuation { continuation in + SFSpeechRecognizer.requestAuthorization { status in + continuation.resume(returning: status == .authorized) + } + } + } + + func start(localeIdentifier: String) throws { + // Only a normal empty final can prepare this one-shot handoff. Taking + // it before cleanup preserves new speech received after endAudio. + let carriedAudio = handoff.takeReadyAudio(now: ProcessInfo.processInfo.systemUptime) + stop(reason: "restart") + guard SFSpeechRecognizer.authorizationStatus() == .authorized else { + recordFailure(.permissionDenied) + throw Failure.permissionDenied + } + guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)), + recognizer.supportsOnDeviceRecognition else { + recordFailure(.languageUnavailable) + throw Failure.languageUnavailable + } + guard recognizer.isAvailable else { + recordFailure(.temporarilyUnavailable) + throw Failure.temporarilyUnavailable + } + let request = SFSpeechAudioBufferRecognitionRequest() + request.requiresOnDeviceRecognition = true + request.shouldReportPartialResults = true + request.taskHint = .dictation + if #available(iOS 16.0, *) { request.addsPunctuation = true } + self.recognizer = recognizer + self.request = request + latestText = "" + ending = false + startedAt = ProcessInfo.processInfo.systemUptime + lastPacketAt = startedAt + lastVoiceAt = nil + lastTranscriptAt = nil + isListening = true + diagnostics.started() + logDiagnostics(event: "started") + let id = sessionID + task = recognizer.recognitionTask(with: request) { [weak self] result, error in + Task { @MainActor in + guard let self, self.sessionID == id else { return } + if let result { + let text = result.bestTranscription.formattedString + if !text.isEmpty && text != self.latestText { + self.lastVoiceAt = ProcessInfo.processInfo.systemUptime + self.lastTranscriptAt = self.lastVoiceAt + } + self.latestText = text + if !result.isFinal { + self.diagnostics.partial(nonempty: !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + self.onPartial?(self.latestText) + if result.isFinal { + self.complete(reason: "native_final", allowHandoff: error == nil) + return + } + } + if let error { + if self.ending && !self.latestText.isEmpty { + self.complete(reason: "finalization_error", allowHandoff: false) + } else { + let failure: Failure + if SFSpeechRecognizer.authorizationStatus() != .authorized { + failure = .permissionDenied + } else if self.recognizer?.supportsOnDeviceRecognition == false { + failure = .languageUnavailable + } else { + failure = .recognitionInterrupted(error.localizedDescription) + } + self.stop(reason: "recognizer_error") + self.recordFailure(failure, underlying: error) + self.onError?(failure) + } + } + } + } + timer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: true) { [weak self] _ in + Task { @MainActor in self?.tick() } + } + if !carriedAudio.isEmpty { appendPCM16LE(carriedAudio) } + onStatus?("正在听 Rokid 麦克风（离线识别）") + } + + func appendPCM16LE(_ data: Data) { + guard !data.isEmpty else { return } + if ending { + // This is only the short endAudio/final-result window. Controller + // gates still discard all inference, TTS and user-paused audio. + handoff.append(data) + return + } + guard isListening, let request else { return } + // Preserve a split sample across websocket messages. At most one byte + // is retained; audio received during inference/TTS is simply dropped. + var bytes = [UInt8]() + bytes.reserveCapacity(data.count + 1) + if let pendingByte { bytes.append(pendingByte) } + bytes.append(contentsOf: data) + pendingByte = bytes.count.isMultiple(of: 2) ? nil : bytes.removeLast() + let frames = bytes.count / 2 + guard frames > 0, + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames)), + let samples = buffer.floatChannelData?[0] else { return } + buffer.frameLength = AVAudioFrameCount(frames) + var squareSum: Float = 0 + for index in 0.. 0.012 { lastVoiceAt = now } + request.append(buffer) + diagnostics.appended(samples: frames, rms: Double(rms)) + } + + /// Stop accepting audio and give the recognizer time to finalize its text. + func finishUtterance(reason: String = "manual") { + guard isListening, !ending else { return } + ending = true + isListening = false + handoff.begin() + // A split sample's first byte has not reached the old request. Keep it + // ahead of the post-endAudio stream so the next request stays aligned. + if let pendingByte { handoff.append(Data([pendingByte])) } + pendingByte = nil + diagnostics.finishing(reason: reason) + logDiagnostics(event: "finishing") + timer?.invalidate() + timer = nil + request?.endAudio() + onStatus?("正在结束语句…") + let id = sessionID + finishTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: false) { [weak self] _ in + Task { @MainActor in + guard let self, self.sessionID == id else { return } + self.complete(reason: "finish_timeout") + } + } + } + + func stop(reason: String = "cancelled") { + cancelRecognition(reason: reason, preservingHandoff: false) + } + + private func cancelRecognition(reason: String, preservingHandoff: Bool) { + if !preservingHandoff { handoff.clear() } + let wasActive = diagnostics.state != .idle + // No-op cleanup after a final/error must not replace its useful cause. + if wasActive { diagnostics.stopped(reason: reason) } + sessionID = UUID() + isListening = false + ending = false + timer?.invalidate() + timer = nil + finishTimer?.invalidate() + finishTimer = nil + request?.endAudio() + task?.cancel() + task = nil + request = nil + recognizer = nil + pendingByte = nil + if wasActive { logDiagnostics(event: "stopped") } + } + + /// Reset only when beginning a new receiver run, never between ASR retries. + func resetDiagnostics() { + diagnostics.reset() + logDiagnostics(event: "reset") + } + + private func tick() { + guard isListening else { return } + let now = ProcessInfo.processInfo.systemUptime + let action = OpenGlassUtteranceTiming.action(now: now, startedAt: startedAt, + lastPacketAt: lastPacketAt, lastVoiceAt: lastVoiceAt, + lastTranscriptAt: lastTranscriptAt, hasText: containsWords(latestText)) + switch action { + case .finish: finishUtterance(reason: "timer_finish") + case .audioTimeout: + stop(reason: "audio_timeout") + recordFailure(.audioTimeout) + onError?(.audioTimeout) + case .listen: break + } + } + + private func complete(reason: String, allowHandoff: Bool = true) { + let candidate = latestText.trimmingCharacters(in: .whitespacesAndNewlines) + // Punctuation-only final results are not questions. A normal empty + // result lets the controller rearm immediately when fresh PCM is ready. + let text = containsWords(candidate) ? candidate : "" + let preserveHandoff = text.isEmpty && allowHandoff + handoff.complete(empty: preserveHandoff, now: ProcessInfo.processInfo.systemUptime) + diagnostics.completed(nonempty: !text.isEmpty, reason: reason) + cancelRecognition(reason: reason, preservingHandoff: preserveHandoff) + logDiagnostics(event: "final") + onFinal?(text) + } + + private func containsWords(_ text: String) -> Bool { + text.unicodeScalars.contains { CharacterSet.alphanumerics.contains($0) } + } + + private func recordFailure(_ failure: Failure, underlying: Error? = nil) { + let reason: String + let localCode: Int + switch failure { + case .permissionDenied: reason = "permission_denied"; localCode = 1 + case .languageUnavailable: reason = "language_unavailable"; localCode = 2 + case .temporarilyUnavailable: reason = "recognizer_unavailable"; localCode = 3 + case .audioTimeout: reason = "audio_timeout"; localCode = 4 + case .recognitionInterrupted: reason = "recognizer_error"; localCode = 5 + } + let nativeError = underlying.map { $0 as NSError } + diagnostics.failed(domain: nativeError?.domain ?? "OpenGlassSpeech", + code: nativeError?.code ?? localCode, reason: reason) + logDiagnostics(event: "failed") + } + + /// State transitions only: never log PCM, recognition text, or error descriptions. + private func logDiagnostics(event: String) { + let d = diagnostics + let summary = "OpenGlassASR event=\(event) state=\(d.state.rawValue) session=\(d.sessions)" + + " accepted_samples=\(d.acceptedSamples) session_samples=\(d.sessionSamples)" + + " rms=\(String(format: "%.5f", d.lastRMS)) peak_rms=\(String(format: "%.5f", d.sessionPeakRMS))" + + " partials=\(d.partialCount) nonempty_partials=\(d.nonemptyPartialCount)" + + " empty_finals=\(d.emptyFinalCount) nonempty_finals=\(d.nonemptyFinalCount)" + + " failures=\(d.failureCount) failure_domain=\(d.lastFailureDomain)" + + " failure_code=\(d.lastFailureCode.map(String.init) ?? "none") end=\(d.lastEndReason)" + NSLog("%@", summary) + } + + enum Failure: LocalizedError { + case permissionDenied, languageUnavailable, temporarilyUnavailable, audioTimeout + case recognitionInterrupted(String) + + var requiresUserAction: Bool { + switch self { + case .permissionDenied, .languageUnavailable: return true + default: return false + } + } + var errorDescription: String? { + switch self { + case .permissionDenied: + return "未获语音识别权限。请在系统设置中允许后点击“恢复自动语音”；文字提问仍可使用。" + case .languageUnavailable: + return "此语言的离线识别资源不可用。请下载资源或切换语言后恢复；不会转到云端。" + case .temporarilyUnavailable: + return "系统离线识别暂不可用。" + case .audioTimeout: + return "Rokid 音频暂时没有到达。" + case .recognitionInterrupted(let message): + return "离线识别暂时中断：\(message)" + } + } + } +} diff --git a/ios/Sources/OpenGlassViewController.swift b/ios/Sources/OpenGlassViewController.swift new file mode 100644 index 0000000..ab1fc80 --- /dev/null +++ b/ios/Sources/OpenGlassViewController.swift @@ -0,0 +1,931 @@ +import UIKit +import AVFoundation +import Darwin + +/// Foreground Rokid receiver with one-image, one-question local inference. +@MainActor +final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSpeechSynthesizerDelegate { + private let server = RokidIngressServer() + private let model = OpenGlassModelSession() + // The upstream downloader requires its ObservableObject for file/progress + // bookkeeping. Never call initialize() on this download-only wrapper. + private lazy var downloadWrapper = MTMDWrapperExample() + private let speech = OpenGlassSpeechSession() + private let synthesizer = AVSpeechSynthesizer() + private var activeUtterance: AVSpeechUtterance? + private var isRunning = false + private var isStarting = false + private var isAuthorizingSpeech = false + private var isAnswering = false + private var audioConnected = false + private var waitForFreshAudio = false + private var voiceLoop = OpenGlassVoiceLoopPolicy() + private var wantsService = false + private var attemptedAutomaticStart = false + private var isInBackground = false + private var audioInterruptionActive = false + private var listeningWakeTask: Task? + private var speechAuthorizationID = UUID() + private var completedTurns = 0 + private var suppressAnswerSpeech = false + private var runID = UUID() + private var turnID = UUID() + private var latestRawImage: Data? + private var latestImage: UIImage? + private var latestImageAt: Date? + private var frameTimer: Timer? + private var backgroundTask: UIBackgroundTaskIdentifier = .invalid + private let maximumFrameAge: TimeInterval = 8 + + private let statusLabel = UILabel() + private let modelLabel = UILabel() + private let connectionLabel = UILabel() + private let frameLabel = UILabel() + private let routeLabel = UILabel() + private let voiceStatusLabel = UILabel() + private let preview = UIImageView() + private let addressView = UITextView() + private let transcriptLabel = UILabel() + private let answerView = UITextView() + private let questionField = UITextField() + private let serviceButton = UIButton(type: .system) + private let askButton = UIButton(type: .system) + private let listenButton = UIButton(type: .system) + private let stopTurnButton = UIButton(type: .system) + private let voiceSwitch = UISwitch() + private let speakSwitch = UISwitch() + private let rotationControl = UISegmentedControl(items: ["0°", "90°", "180°", "270°"]) + private let languageControl = UISegmentedControl(items: ["中文", "English"]) + private var downloadItem: UIBarButtonItem! + + override func viewDidLoad() { + super.viewDidLoad() + title = "OpenGlass · iPhone" + view.backgroundColor = .systemBackground + buildUI() + connectCallbacks() + synthesizer.delegate = self + NotificationCenter.default.addObserver(self, selector: #selector(enteredBackground), + name: UIApplication.didEnterBackgroundNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(enteredForeground), + name: UIApplication.willEnterForegroundNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(audioInterrupted), + name: AVAudioSession.interruptionNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(updateAudioRoute), + name: AVAudioSession.routeChangeNotification, object: nil) + statusLabel.text = "准备就绪。保持 App 在前台，点击启动。" + modelLabel.text = "固定模型：MiniCPM-V 4.6 · 本机推理" + connectionLabel.text = "Rokid 尚未连接" + answerView.text = "回答会显示在这里。" + frameTimer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in + Task { @MainActor in + self?.refreshFrameAge() + self?.refreshAddresses() + self?.reconcileAutomaticLoop() + self?.refreshVoiceStatus() + } + } + refreshAddresses() + refreshFrameAge() + updateAudioRoute() + updateControls() + } + + override func viewDidAppear(_ animated: Bool) { + super.viewDidAppear(animated) + refreshAddresses() + if !attemptedAutomaticStart { + attemptedAutomaticStart = true + let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] + let hasModelFiles = OpenGlassModelSession.modelFiles.allSatisfy { + FileManager.default.fileExists(atPath: documents.appendingPathComponent($0).path) + } + if hasModelFiles { + wantsService = true + if voiceSwitch.isOn { voiceLoop.requestListening() } + reconcileAutomaticLoop() + } else { + statusLabel.text = "首次使用请在“模型下载”中准备 MiniCPM-V 4.6，然后启动接收。" + } + } + UIApplication.shared.isIdleTimerDisabled = isRunning + } + + deinit { + frameTimer?.invalidate() + listeningWakeTask?.cancel() + NotificationCenter.default.removeObserver(self) + } + + private func buildUI() { + downloadItem = UIBarButtonItem(title: "模型下载", style: .plain, target: self, action: #selector(showDownloads)) + navigationItem.rightBarButtonItem = downloadItem + let scroll = UIScrollView() + scroll.keyboardDismissMode = .interactive + scroll.translatesAutoresizingMaskIntoConstraints = false + view.addSubview(scroll) + let stack = UIStackView() + stack.axis = .vertical + stack.spacing = 12 + stack.translatesAutoresizingMaskIntoConstraints = false + scroll.addSubview(stack) + NSLayoutConstraint.activate([ + scroll.leadingAnchor.constraint(equalTo: view.safeAreaLayoutGuide.leadingAnchor), + scroll.trailingAnchor.constraint(equalTo: view.safeAreaLayoutGuide.trailingAnchor), + scroll.topAnchor.constraint(equalTo: view.safeAreaLayoutGuide.topAnchor), + scroll.bottomAnchor.constraint(equalTo: view.keyboardLayoutGuide.topAnchor), + stack.leadingAnchor.constraint(equalTo: scroll.contentLayoutGuide.leadingAnchor, constant: 18), + stack.trailingAnchor.constraint(equalTo: scroll.contentLayoutGuide.trailingAnchor, constant: -18), + stack.topAnchor.constraint(equalTo: scroll.contentLayoutGuide.topAnchor, constant: 16), + stack.bottomAnchor.constraint(equalTo: scroll.contentLayoutGuide.bottomAnchor, constant: -24), + stack.widthAnchor.constraint(equalTo: scroll.frameLayoutGuide.widthAnchor, constant: -36) + ]) + for label in [statusLabel, modelLabel, connectionLabel, frameLabel, routeLabel, voiceStatusLabel, transcriptLabel] { + label.numberOfLines = 0 + label.font = .preferredFont(forTextStyle: .subheadline) + label.adjustsFontForContentSizeCategory = true + } + statusLabel.font = .preferredFont(forTextStyle: .headline) + modelLabel.textColor = .secondaryLabel + frameLabel.textColor = .secondaryLabel + routeLabel.textColor = .secondaryLabel + voiceStatusLabel.textColor = .secondaryLabel + stack.addArrangedSubview(statusLabel) + stack.addArrangedSubview(modelLabel) + configureButton(serviceButton, title: "启动接收", action: #selector(toggleService), filled: true) + stack.addArrangedSubview(serviceButton) + addressView.isEditable = false + addressView.isScrollEnabled = false + addressView.font = .monospacedSystemFont(ofSize: 13, weight: .regular) + addressView.backgroundColor = .secondarySystemBackground + addressView.layer.cornerRadius = 8 + addressView.accessibilityLabel = "iPhone 接收地址" + stack.addArrangedSubview(addressView) + let addressRow = UIStackView() + addressRow.axis = .horizontal + addressRow.spacing = 12 + let copy = UIButton(type: .system) + configureButton(copy, title: "复制地址", action: #selector(copyAddress)) + let refresh = UIButton(type: .system) + configureButton(refresh, title: "刷新地址", action: #selector(refreshAddresses)) + addressRow.addArrangedSubview(copy) + addressRow.addArrangedSubview(refresh) + stack.addArrangedSubview(addressRow) + stack.addArrangedSubview(connectionLabel) + preview.contentMode = .scaleAspectFit + preview.backgroundColor = .secondarySystemBackground + preview.layer.cornerRadius = 12 + preview.clipsToBounds = true + preview.accessibilityLabel = "最新 Rokid 画面" + preview.heightAnchor.constraint(equalToConstant: 220).isActive = true + stack.addArrangedSubview(preview) + stack.addArrangedSubview(frameLabel) + let savedRotation = UserDefaults.standard.object(forKey: "OpenGlass.imageRotation") as? Int ?? 270 + rotationControl.selectedSegmentIndex = [0, 90, 180, 270].firstIndex(of: savedRotation) ?? 3 + rotationControl.addTarget(self, action: #selector(rotationChanged), for: .valueChanged) + rotationControl.accessibilityLabel = "画面顺时针旋转" + stack.addArrangedSubview(row("画面旋转", rotationControl)) + voiceSwitch.isOn = UserDefaults.standard.object(forKey: "OpenGlass.voiceEnabled") as? Bool ?? true + voiceSwitch.addTarget(self, action: #selector(voiceChanged), for: .valueChanged) + stack.addArrangedSubview(row("Rokid 离线语音提问", voiceSwitch)) + languageControl.selectedSegmentIndex = UserDefaults.standard.integer(forKey: "OpenGlass.speechLanguage") + if !(0...1).contains(languageControl.selectedSegmentIndex) { languageControl.selectedSegmentIndex = 0 } + languageControl.addTarget(self, action: #selector(languageChanged), for: .valueChanged) + stack.addArrangedSubview(row("识别 / 播报语言", languageControl)) + speakSwitch.isOn = UserDefaults.standard.object(forKey: "OpenGlass.speakAnswers") as? Bool ?? true + speakSwitch.addTarget(self, action: #selector(speakChanged), for: .valueChanged) + stack.addArrangedSubview(row("朗读回答", speakSwitch)) + stack.addArrangedSubview(routeLabel) + stack.addArrangedSubview(voiceStatusLabel) + configureButton(listenButton, title: "恢复自动语音", action: #selector(toggleListening)) + stack.addArrangedSubview(listenButton) + transcriptLabel.text = "启动后自动听取 Rokid 语音。停顿约1.3秒后提问，朗读结束会自动听下一句。" + stack.addArrangedSubview(transcriptLabel) + questionField.borderStyle = .roundedRect + questionField.placeholder = "输入问题，例如：我面前有什么？" + questionField.returnKeyType = .send + questionField.delegate = self + questionField.accessibilityLabel = "文字问题" + questionField.heightAnchor.constraint(greaterThanOrEqualToConstant: 44).isActive = true + stack.addArrangedSubview(questionField) + configureButton(askButton, title: "用最新画面提问", action: #selector(askManually), filled: true) + configureButton(stopTurnButton, title: "停止本轮 / 暂停语音", action: #selector(stopCurrentTurn)) + stack.addArrangedSubview(askButton) + stack.addArrangedSubview(stopTurnButton) + answerView.isEditable = false + answerView.isScrollEnabled = false + answerView.font = .preferredFont(forTextStyle: .body) + answerView.backgroundColor = .secondarySystemBackground + answerView.layer.cornerRadius = 8 + answerView.heightAnchor.constraint(greaterThanOrEqualToConstant: 130).isActive = true + answerView.accessibilityLabel = "模型回答" + stack.addArrangedSubview(answerView) + let note = UILabel() + note.numberOfLines = 0 + note.font = .preferredFont(forTextStyle: .footnote) + note.textColor = .secondaryLabel + note.text = "Rokid 可通过同一局域网或 Windows 双 USB 中继连接。首次使用请先下载模型。此 App 的模型文件与原 MiniCPM 示例分开保存。进入后台会停止接收和语音；回到前台会恢复此前运行状态。主动停止后不会自动重启。" + stack.addArrangedSubview(note) + } + + private func row(_ title: String, _ control: UIView) -> UIStackView { + let label = UILabel() + label.text = title + label.font = .preferredFont(forTextStyle: .subheadline) + label.numberOfLines = 0 + let stack = UIStackView(arrangedSubviews: [label, control]) + stack.axis = .horizontal + stack.spacing = 12 + stack.alignment = .center + control.setContentHuggingPriority(.required, for: .horizontal) + return stack + } + + private func configureButton(_ button: UIButton, title: String, action: Selector, filled: Bool = false) { + var configuration = filled ? UIButton.Configuration.filled() : UIButton.Configuration.tinted() + configuration.title = title + configuration.cornerStyle = .medium + button.configuration = configuration + button.heightAnchor.constraint(greaterThanOrEqualToConstant: 44).isActive = true + button.addTarget(self, action: action, for: .touchUpInside) + } + + private func connectCallbacks() { + server.onImage = { [weak self] data in + guard let self, self.isRunning, let decoded = UIImage(data: data) else { return } + self.latestRawImage = data + self.latestImage = self.normalizedImage(decoded) + self.latestImageAt = Date() + self.preview.image = self.latestImage + self.refreshFrameAge() + self.reconcileAutomaticLoop() + } + server.onAudio = { [weak self] data in + guard let self else { return } + // An empty final with no handoff waits for actual PCM, not a timer + // or image callback. This avoids spinning on immediate empty tasks. + if !data.isEmpty { self.waitForFreshAudio = false } + self.reconcileAutomaticLoop() + guard self.isRunning, !self.isInBackground, !self.audioInterruptionActive, + !self.isAnswering, !self.model.isBusy, self.activeUtterance == nil, + self.voiceLoop.canListen(at: self.monotonicNow) else { return } + self.speech.appendPCM16LE(data) + } + server.onStatus = { [weak self] status in self?.connectionLabel.text = status } + server.onFailure = { [weak self] message in + self?.stopService(message: "接收服务失败：\(message)") + } + server.onAudioConnectionChanged = { [weak self] connected in + guard let self else { return } + self.audioConnected = connected + if !connected { + self.waitForFreshAudio = false + self.speech.stop(reason: "audio_disconnected") + if self.isRunning { self.statusLabel.text = "Rokid 音频已断开，等待重新连接。" } + } else { + self.reconcileAutomaticLoop() + } + self.updateControls() + } + model.onChange = { [weak self] _, message in + self?.modelLabel.text = message + self?.updateControls() + } + model.onOutput = { [weak self] output in self?.answerView.text = output } + speech.onPartial = { [weak self] text in self?.transcriptLabel.text = text } + speech.onStatus = { [weak self] message in + self?.statusLabel.text = message + self?.updateControls() + } + speech.onError = { [weak self] failure in + self?.handleSpeechFailure(failure) + } + speech.onFinal = { [weak self] text in + guard let self, self.isRunning, !self.isInBackground, + self.voiceLoop.wantsListening, self.voiceLoop.blockedReason == nil else { return } + if text.isEmpty { + // Silence is normal in a continuous conversation. Backoff is + // reserved for real errors; it otherwise drops the next words. + self.voiceLoop.receivedUtterance() + self.cancelListeningWake() + self.waitForFreshAudio = !self.speech.hasPendingHandoffAudio + self.statusLabel.text = "继续等待 Rokid 语音。" + self.reconcileAutomaticLoop() + } else { + self.waitForFreshAudio = false + self.voiceLoop.receivedUtterance() + self.questionField.text = text + self.submit(text, fromSpeech: true) + } + self.updateControls() + } + } + + @objc private func toggleService() { + if wantsService || isRunning || isStarting { + stopService(message: "接收已停止，点击启动可恢复。") + return + } + guard !model.requiresAppRestart else { return } + wantsService = true + if voiceSwitch.isOn { voiceLoop.requestListening() } else { voiceLoop.pause() } + reconcileAutomaticLoop() + } + + private func startServiceIfPossible() { + guard wantsService, !isInBackground, !isRunning, !isStarting, + !model.isBusy, !isAnswering, !model.requiresAppRestart else { return } + isStarting = true + isRunning = true + completedTurns = 0 + waitForFreshAudio = false + speech.resetDiagnostics() + runID = UUID() + let id = runID + latestRawImage = nil + latestImage = nil + latestImageAt = nil + preview.image = nil + do { + try server.start(port: 18080) + } catch { + stopService(message: "无法启动接收：\(error.localizedDescription)") + return + } + UIApplication.shared.isIdleTimerDisabled = true + statusLabel.text = "接收已启动，正在准备模型…" + refreshAddresses() + refreshFrameAge() + updateControls() + Task { + guard isRunning, runID == id, !isInBackground else { return } + do { + try await model.load() + guard isRunning, runID == id else { updateControls(); return } + statusLabel.text = voiceLoop.wantsListening + ? "等待 Rokid 画面和音频；也可以文字提问。" + : "接收已启动；自动语音保持关闭或暂停，仍可文字提问。" + if voiceSwitch.isOn, voiceLoop.wantsListening, voiceLoop.blockedReason == nil { + let authorizationID = UUID() + speechAuthorizationID = authorizationID + isAuthorizingSpeech = true + let authorized = await OpenGlassSpeechSession.authorize() + guard isRunning, runID == id else { updateControls(); return } + if speechAuthorizationID == authorizationID { + isAuthorizingSpeech = false + if !authorized, voiceLoop.wantsListening { + handleSpeechFailure(.permissionDenied) + } + } + } + isStarting = false + reconcileAutomaticLoop() + } catch { + if runID == id { stopService(message: error.localizedDescription) } + } + updateControls() + endBackgroundTaskIfIdle() + } + } + + private func stopService(message: String, preservingIntent: Bool = false) { + if !preservingIntent { + wantsService = false + voiceLoop.pause() + } + isRunning = false + isStarting = false + isAuthorizingSpeech = false + speechAuthorizationID = UUID() + cancelListeningWake() + waitForFreshAudio = false + suppressAnswerSpeech = true + runID = UUID() + turnID = UUID() + speech.stop(reason: preservingIntent ? "background" : "receiver_stopped") + stopSpeaking() + server.stop() + audioConnected = false + model.requestStop() + UIApplication.shared.isIdleTimerDisabled = false + statusLabel.text = model.isBusy ? "已停止接收和语音；正在结束本轮推理…" : message + updateControls() + } + + @objc private func stopCurrentTurn() { + turnID = UUID() + voiceLoop.pause() + cancelListeningWake() + isAuthorizingSpeech = false + speechAuthorizationID = UUID() + suppressAnswerSpeech = true + waitForFreshAudio = false + speech.stop(reason: "user_paused") + stopSpeaking() + model.requestStop() + statusLabel.text = model.isBusy ? "正在结束本轮推理…" : "语音已暂停，点击“恢复自动语音”可恢复。" + updateControls() + } + + @objc private func askManually() { + questionField.resignFirstResponder() + submit(questionField.text ?? "") + } + + func textFieldShouldReturn(_ textField: UITextField) -> Bool { + askManually() + return true + } + + private func submit(_ text: String, fromSpeech: Bool = false) { + let question = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !question.isEmpty else { statusLabel.text = "请先输入或说出问题。"; return } + guard isRunning, !isInBackground, !isStarting, model.isLoaded, !model.isBusy, !isAnswering else { + statusLabel.text = "请先启动接收，并等待当前模型任务结束。" + return + } + guard question.count <= 500 else { + if fromSpeech { retryListening(message: "问题超过500字，请简短地重新说一次。") } + else { statusLabel.text = "问题请控制在500字以内。" } + return + } + guard let image = latestImage, let receivedAt = latestImageAt, + Date().timeIntervalSince(receivedAt) <= maximumFrameAge else { + speech.stop() + statusLabel.text = "本句未提交：没有8秒内的新画面。收到新画面后会自动继续听，请重新说出问题。" + updateControls() + return + } + waitForFreshAudio = false + speech.stop(reason: "model_submission") + stopSpeaking() + isAnswering = true + suppressAnswerSpeech = false + answerView.text = "" + statusLabel.text = "已暂停收音，正在本机理解画面…" + let id = runID + turnID = UUID() + let currentTurnID = turnID + updateControls() + Task { + guard isRunning, runID == id, turnID == currentTurnID, !isInBackground else { + isAnswering = false + updateControls() + reconcileAutomaticLoop() + return + } + var imageURL: URL? + do { + let folder = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent("OpenGlassFrames", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let file = folder.appendingPathComponent(UUID().uuidString + ".jpg") + guard let jpeg = image.jpegData(compressionQuality: 0.85) else { + throw OpenGlassModelSession.SessionError.invalidFile("当前画面") + } + try jpeg.write(to: file, options: .atomic) + imageURL = file + let answer = try await model.ask(question: question, imageURL: file, answerLanguage: localeIdentifier) + isAnswering = false + if isRunning, runID == id, turnID == currentTurnID { + completedTurns += 1 + answerView.text = answer.isEmpty ? "模型没有返回可朗读的文字，请换个问题重试。" : answer + if !suppressAnswerSpeech, speakSwitch.isOn, !answer.isEmpty { + speak(answer) + } else { + statusLabel.text = "回答完成。" + resumeListeningAfterEchoDelay() + } + } + } catch { + isAnswering = false + if isRunning, runID == id { + if model.requiresAppRestart || !model.isLoaded { + stopService(message: error.localizedDescription) + } else if turnID == currentTurnID, voiceLoop.wantsListening { + retryListening(message: error.localizedDescription) + } + } + } + // A timed-out native prefill may still read its file and context. + if !model.requiresAppRestart, let imageURL { try? FileManager.default.removeItem(at: imageURL) } + updateControls() + endBackgroundTaskIfIdle() + reconcileAutomaticLoop() + } + } + + private var localeIdentifier: String { languageControl.selectedSegmentIndex == 1 ? "en-US" : "zh-CN" } + private var monotonicNow: TimeInterval { ProcessInfo.processInfo.systemUptime } + private var hasFreshFrame: Bool { + guard latestImage != nil, let date = latestImageAt else { return false } + return Date().timeIntervalSince(date) <= maximumFrameAge + } + + /// All timer, frame, connection and audio callbacks go through the same + /// admission gate. None can bypass a retry or the post-TTS echo deadline. + private func reconcileAutomaticLoop() { + guard !isInBackground else { return } + if !isRunning { startServiceIfPossible() } + startListeningIfPossible() + } + + private func startListeningIfPossible() { + guard isRunning, !isInBackground, !audioInterruptionActive, + !isStarting, !isAuthorizingSpeech, !isAnswering, !model.isBusy, model.isLoaded, + !model.requiresAppRestart, voiceSwitch.isOn, voiceLoop.wantsListening, + voiceLoop.blockedReason == nil, audioConnected, hasFreshFrame, + activeUtterance == nil, !speech.isActive, !waitForFreshAudio else { return } + guard voiceLoop.canListen(at: monotonicNow) else { + scheduleListeningWake() + return + } + do { + try speech.start(localeIdentifier: localeIdentifier) + } catch let failure as OpenGlassSpeechSession.Failure { + handleSpeechFailure(failure) + } catch { + retryListening(message: error.localizedDescription) + } + updateControls() + } + + private func handleSpeechFailure(_ failure: OpenGlassSpeechSession.Failure) { + guard isRunning, voiceLoop.wantsListening else { return } + speech.stop() + if failure.requiresUserAction { + voiceLoop.block(reason: failure.localizedDescription) + cancelListeningWake() + statusLabel.text = failure.localizedDescription + updateControls() + } else { + retryListening(message: failure.localizedDescription) + } + } + + private func retryListening(message: String) { + guard isRunning, !isInBackground, voiceLoop.wantsListening, + voiceLoop.blockedReason == nil else { return } + speech.stop() + let delay = voiceLoop.retry(now: monotonicNow) + statusLabel.text = "\(message) \(delay.formatted())秒后自动重试。" + scheduleListeningWake() + updateControls() + } + + private func cancelListeningWake() { + listeningWakeTask?.cancel() + listeningWakeTask = nil + } + + private func scheduleListeningWake() { + guard listeningWakeTask == nil, isRunning, !isInBackground, + voiceLoop.wantsListening, voiceLoop.blockedReason == nil else { return } + let delay = voiceLoop.notBefore - monotonicNow + guard delay > 0 else { return } + let id = runID + listeningWakeTask = Task { [weak self] in + do { try await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) } + catch { return } + guard let self, !Task.isCancelled, self.runID == id else { return } + self.listeningWakeTask = nil + self.reconcileAutomaticLoop() + } + } + + @objc private func toggleListening() { + if speech.isListening { speech.finishUtterance(); updateControls(); return } + if speech.isFinishing { return } + guard isRunning, !isAuthorizingSpeech else { return } + voiceSwitch.isOn = true + UserDefaults.standard.set(true, forKey: "OpenGlass.voiceEnabled") + voiceLoop.requestListening() + waitForFreshAudio = false + isAuthorizingSpeech = true + let id = runID + let authorizationID = UUID() + speechAuthorizationID = authorizationID + updateControls() + Task { + let authorized = await OpenGlassSpeechSession.authorize() + guard isRunning, runID == id, speechAuthorizationID == authorizationID else { return } + isAuthorizingSpeech = false + if authorized { + if !audioConnected { statusLabel.text = "等待 Rokid 音频连接…" } + else if !hasFreshFrame { statusLabel.text = "等待新画面，随后自动听取语音。" } + reconcileAutomaticLoop() + } else { + handleSpeechFailure(.permissionDenied) + } + updateControls() + } + } + + @objc private func voiceChanged() { + UserDefaults.standard.set(voiceSwitch.isOn, forKey: "OpenGlass.voiceEnabled") + if voiceSwitch.isOn { + if isRunning { toggleListening() } + } else { + waitForFreshAudio = false + speech.stop(reason: "voice_disabled") + voiceLoop.pause() + cancelListeningWake() + isAuthorizingSpeech = false + speechAuthorizationID = UUID() + statusLabel.text = "自动语音已关闭，仍可文字提问。" + } + updateControls() + } + + @objc private func languageChanged() { + UserDefaults.standard.set(languageControl.selectedSegmentIndex, forKey: "OpenGlass.speechLanguage") + waitForFreshAudio = false + speech.stop(reason: "language_changed") + if voiceLoop.wantsListening { voiceLoop.requestListening() } + reconcileAutomaticLoop() + } + + @objc private func speakChanged() { + UserDefaults.standard.set(speakSwitch.isOn, forKey: "OpenGlass.speakAnswers") + if !speakSwitch.isOn { + stopSpeaking() + resumeListeningAfterEchoDelay() + } + } + + private func speak(_ text: String) { + speech.stop(reason: "tts_started") + do { + let audio = AVAudioSession.sharedInstance() + try audio.setCategory(.playback, mode: .spokenAudio, options: []) + try audio.setActive(true) + let utterance = AVSpeechUtterance(string: text) + utterance.voice = AVSpeechSynthesisVoice(language: localeIdentifier) + utterance.rate = AVSpeechUtteranceDefaultSpeechRate + activeUtterance = utterance + statusLabel.text = "正在朗读；收音已暂停。" + updateAudioRoute() + synthesizer.speak(utterance) + } catch { + activeUtterance = nil + statusLabel.text = "朗读不可用：\(error.localizedDescription)。回答已保留在屏幕。" + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + resumeListeningAfterEchoDelay() + } + updateControls() + } + + private func stopSpeaking() { + if activeUtterance != nil || synthesizer.isSpeaking { + voiceLoop.protectFromEcho(now: monotonicNow) + } + // Invalidate identity before stopSpeaking can dispatch didCancel. + activeUtterance = nil + synthesizer.stopSpeaking(at: .immediate) + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + Task { @MainActor [weak self] in + guard let self, self.activeUtterance === utterance else { return } + self.activeUtterance = nil + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + self.statusLabel.text = "朗读完成。" + self.resumeListeningAfterEchoDelay() + self.updateControls() + } + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { + Task { @MainActor [weak self] in + guard let self, self.activeUtterance === utterance else { return } + self.activeUtterance = nil + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + self.statusLabel.text = "朗读被中断，稍后自动继续听取语音。" + self.resumeListeningAfterEchoDelay() + self.updateControls() + } + } + + private func resumeListeningAfterEchoDelay() { + voiceLoop.protectFromEcho(now: monotonicNow) + scheduleListeningWake() + reconcileAutomaticLoop() + } + + @objc private func rotationChanged() { + UserDefaults.standard.set(rotationControl.selectedSegmentIndex * 90, forKey: "OpenGlass.imageRotation") + if let data = latestRawImage, let image = UIImage(data: data) { + latestImage = normalizedImage(image) + preview.image = latestImage + } + } + + private func normalizedImage(_ image: UIImage) -> UIImage { + let quarterTurns = rotationControl.selectedSegmentIndex + let radians = CGFloat(quarterTurns) * .pi / 2 + let original = image.size + let swapped = quarterTurns % 2 != 0 + let width = swapped ? original.height : original.width + let height = swapped ? original.width : original.height + let scale = min(1, 1024 / max(width, height)) + let size = CGSize(width: max(1, width * scale), height: max(1, height * scale)) + let format = UIGraphicsImageRendererFormat() + format.scale = 1 + format.opaque = true + return UIGraphicsImageRenderer(size: size, format: format).image { context in + context.cgContext.translateBy(x: size.width / 2, y: size.height / 2) + context.cgContext.rotate(by: radians) + image.draw(in: CGRect(x: -original.width * scale / 2, y: -original.height * scale / 2, + width: original.width * scale, height: original.height * scale)) + } + } + + private func refreshFrameAge() { + if let date = latestImageAt { + let age = max(0, Date().timeIntervalSince(date)) + frameLabel.text = age <= maximumFrameAge + ? "最新画面：\(Int(age)) 秒前 · 提问时使用这一张" + : "画面已过期（\(Int(age))秒前），等待 Rokid 新画面。" + frameLabel.textColor = age <= maximumFrameAge ? .secondaryLabel : .systemOrange + } else { + frameLabel.text = "尚未收到 Rokid 画面" + frameLabel.textColor = .secondaryLabel + } + } + + private func updateControls() { + let busy = model.isBusy || isAnswering + serviceButton.configuration?.title = wantsService || isRunning || isStarting ? "停止接收" : "启动接收" + serviceButton.isEnabled = wantsService || isRunning || (!busy && !model.requiresAppRestart) + askButton.isEnabled = isRunning && !isStarting && model.isLoaded && !busy && !model.requiresAppRestart + listenButton.configuration?.title = speech.isFinishing ? "正在结束语句…" + : (speech.isListening ? "结束语句并提问" : "恢复自动语音") + listenButton.isEnabled = isRunning && !isStarting && !busy && activeUtterance == nil + && !model.requiresAppRestart && !speech.isFinishing && !isAuthorizingSpeech + stopTurnButton.isEnabled = isRunning || busy || activeUtterance != nil + downloadItem.isEnabled = !busy && !model.requiresAppRestart + languageControl.isEnabled = !busy && activeUtterance == nil + rotationControl.isEnabled = !busy + voiceSwitch.isEnabled = !isStarting + if isRunning, let reason = voiceLoop.blockedReason { statusLabel.text = reason } + if model.requiresAppRestart { statusLabel.text = OpenGlassModelSession.SessionError.restartRequired.localizedDescription } + refreshVoiceStatus() + } + + /// Called on transitions and once per second so health reports samples + /// actually accepted by ASR, not merely packets received by the socket. + private func refreshVoiceStatus() { + let phase: String + let gate: String + let label: String + if !isRunning || isInBackground { + (phase, gate, label) = ("paused", "receiver_stopped", "接收已停止") + } else if model.isBusy || isAnswering { + (phase, gate, label) = ("thinking", "model", "正在回答") + } else if activeUtterance != nil { + (phase, gate, label) = ("speaking", "tts", "正在朗读，结束后自动听下一句") + } else if !voiceLoop.wantsListening || !voiceSwitch.isOn || voiceLoop.blockedReason != nil { + (phase, gate, label) = ("paused", "voice_paused", "已暂停") + } else if audioInterruptionActive { + (phase, gate, label) = ("waiting", "audio_interruption", "等待系统音频恢复") + } else if isStarting || isAuthorizingSpeech || !model.isLoaded || model.requiresAppRestart { + (phase, gate, label) = ("waiting", "preparing", "正在准备") + } else if speech.isFinishing { + (phase, gate, label) = ("finishing", "finalizing", "正在确认这一句") + } else if speech.isListening { + (phase, gate, label) = ("listening", "accepting_audio", "正在听 Rokid 麦克风") + } else if !audioConnected { + (phase, gate, label) = ("waiting", "no_audio_connection", "等待眼镜声音连接") + } else if !hasFreshFrame { + (phase, gate, label) = ("waiting", "no_fresh_image", "等待新画面") + } else if !voiceLoop.canListen(at: monotonicNow) { + (phase, gate, label) = ("cooldown", "retry_or_echo_delay", "稍后自动继续听取语音") + } else { + (phase, gate, label) = ("waiting", "fresh_audio", "等待眼镜声音") + } + voiceStatusLabel.text = "自动语音：\(label) · 已完成 \(completedTurns) 轮" + server.setConversationStatus(phase: phase, completedTurns: completedTurns, + asr: speech.diagnostics, gate: gate) + } + + @objc private func showDownloads() { + guard !model.isBusy, !isAnswering, !model.requiresAppRestart else { return } + stopService(message: "模型下载完成后，返回本页点击启动。") + Task { + do { + try await model.unload() + navigationController?.pushViewController(MBV46ModelDetailViewController(with: downloadWrapper), animated: true) + } catch { + statusLabel.text = error.localizedDescription + } + updateControls() + } + } + + @objc private func refreshAddresses() { + let addresses = localIPv4Addresses() + let endpoints = addresses.isEmpty + ? "等待 Wi-Fi / 热点地址；USB 中继仍可使用。" + : addresses.map { "http://\($0):18080" }.joined(separator: "\n") + addressView.text = "无线接收：\(RokidIngressServer.serviceName)\n\(endpoints)\n让眼镜加入此 iPhone 热点或同一 Wi-Fi，选择自动连接。" + } + + @objc private func copyAddress() { + if let first = localIPv4Addresses().first { + UIPasteboard.general.string = "http://\(first):18080" + statusLabel.text = "接收地址已复制。把 Rokid 采集端目标设为这个地址。" + } + } + + private func localIPv4Addresses() -> [String] { + var first: UnsafeMutablePointer? + guard getifaddrs(&first) == 0, let first else { return [] } + defer { freeifaddrs(first) } + var addresses: [(String, String)] = [] + var cursor: UnsafeMutablePointer? = first + while let current = cursor { + let interface = current.pointee + defer { cursor = interface.ifa_next } + guard let address = interface.ifa_addr, address.pointee.sa_family == UInt8(AF_INET), + interface.ifa_flags & UInt32(IFF_UP) != 0, + interface.ifa_flags & UInt32(IFF_LOOPBACK) == 0 else { continue } + let name = String(cString: interface.ifa_name) + guard name == "en0" || name.hasPrefix("bridge") || name.hasPrefix("ap") else { continue } + var host = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + if getnameinfo(address, socklen_t(address.pointee.sa_len), &host, socklen_t(host.count), + nil, 0, NI_NUMERICHOST) == 0 { + addresses.append((name, String(cString: host))) + } + } + return addresses.sorted { $0.0 < $1.0 }.map { $0.1 } + } + + @objc private func updateAudioRoute() { + let names = AVAudioSession.sharedInstance().currentRoute.outputs.map { $0.portName }.joined(separator: "、") + routeLabel.text = "朗读输出：\(names.isEmpty ? "iPhone 系统音频" : names)。可在系统中选择蓝牙眼镜。" + } + + @objc private func enteredBackground() { + isInBackground = true + stopService(message: "已进入后台。回到前台后恢复此前运行状态。", preservingIntent: true) + if model.isBusy, backgroundTask == .invalid { + backgroundTask = UIApplication.shared.beginBackgroundTask(withName: "FinishOpenGlassTurn") { [weak self] in + Task { @MainActor in + guard let self else { return } + self.model.requestStop() + self.endBackgroundTask() + } + } + Task { + while model.isBusy && backgroundTask != .invalid { + try? await Task.sleep(nanoseconds: 200_000_000) + } + endBackgroundTask() + } + } + } + + @objc private func enteredForeground() { + isInBackground = false + // Background suspension deactivated our audio session; iOS may omit + // its matching interruption-ended notification while suspended. + audioInterruptionActive = false + refreshAddresses() + if wantsService { + statusLabel.text = model.isBusy ? "正在结束上一轮推理，随后自动恢复接收。" : "已回到前台，正在恢复接收。" + reconcileAutomaticLoop() + } else { + statusLabel.text = "已回到前台，接收保持停止。" + } + updateControls() + } + + @objc private func audioInterrupted(_ notification: Notification) { + guard let rawType = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = AVAudioSession.InterruptionType(rawValue: rawType) else { return } + switch type { + case .began: + audioInterruptionActive = true + waitForFreshAudio = false + speech.stop(reason: "audio_interruption") + stopSpeaking() + if isRunning { statusLabel.text = "系统音频暂时中断，恢复后继续自动语音。" } + case .ended: + audioInterruptionActive = false + resumeListeningAfterEchoDelay() + @unknown default: break + } + updateControls() + } + + private func endBackgroundTaskIfIdle() { + if !model.isBusy { endBackgroundTask() } + } + + private func endBackgroundTask() { + guard backgroundTask != .invalid else { return } + UIApplication.shared.endBackgroundTask(backgroundTask) + backgroundTask = .invalid + } +} diff --git a/ios/Sources/OpenGlassVoiceLoopPolicy.swift b/ios/Sources/OpenGlassVoiceLoopPolicy.swift new file mode 100644 index 0000000..1fd4a33 --- /dev/null +++ b/ios/Sources/OpenGlassVoiceLoopPolicy.swift @@ -0,0 +1,79 @@ +import Foundation + +/// Recovery timing and user intent for the foreground voice loop. +/// Callers supply ProcessInfo.processInfo.systemUptime for every `now` value. +struct OpenGlassVoiceLoopPolicy { + private(set) var wantsListening = false + private(set) var blockedReason: String? + private(set) var retryCount = 0 + private var retryNotBefore: TimeInterval = 0 + private var echoNotBefore: TimeInterval = 0 + + var notBefore: TimeInterval { max(retryNotBefore, echoNotBefore) } + + mutating func requestListening() { + wantsListening = true + blockedReason = nil + receivedUtterance() + } + + mutating func pause() { + wantsListening = false + blockedReason = nil + receivedUtterance() + } + + mutating func block(reason: String) { + blockedReason = reason + } + + /// Returns the retry delay, while `notBefore` also includes any echo hold. + @discardableResult + mutating func retry(now: TimeInterval) -> TimeInterval { + let delays: [TimeInterval] = [0.5, 1, 2, 4, 8] + let delay = delays[min(retryCount, delays.count - 1)] + retryCount = min(retryCount + 1, delays.count) + retryNotBefore = now + delay + return delay + } + + mutating func receivedUtterance() { + retryCount = 0 + retryNotBefore = 0 + } + + mutating func protectFromEcho(now: TimeInterval, delay: TimeInterval = 0.7) { + echoNotBefore = max(echoNotBefore, now + delay) + } + + func canListen(at time: TimeInterval) -> Bool { + wantsListening && blockedReason == nil && time >= notBefore + } +} + +/// Decides when to finalize recognition without depending on Speech or timers. +struct OpenGlassUtteranceTiming { + enum Action: Equatable { + case listen + case finish + case audioTimeout + } + + static func action(now: TimeInterval, startedAt: TimeInterval, + lastPacketAt: TimeInterval, lastVoiceAt: TimeInterval?, + lastTranscriptAt: TimeInterval?, hasText: Bool) -> Action { + if hasText, let lastTranscriptAt, now - lastTranscriptAt >= 2.5 { + return .finish + } + if hasText, let lastVoiceAt, now - lastVoiceAt >= 1.3 { + return .finish + } + if now - startedAt >= 25 { + return .finish + } + if now - lastPacketAt >= 5 { + return .audioTimeout + } + return .listen + } +} diff --git a/ios/Sources/RokidIngressServer.swift b/ios/Sources/RokidIngressServer.swift new file mode 100644 index 0000000..2df3686 --- /dev/null +++ b/ios/Sources/RokidIngressServer.swift @@ -0,0 +1,357 @@ +import Foundation +import Network + +/// Foreground-only local receiver for OpenGlassRokidSensor 0.1.3. +/// Call start/stop on the main queue; all callbacks are delivered on that queue. +final class RokidIngressServer { + // A stable, non-personal name lets the glasses find the same phone after + // DHCP changes. Never advertise the user's device name or account. + static let serviceType = "_openglass._tcp" + static let serviceName: String = { + let key = "OpenGlass.receiverServiceName" + if let saved = UserDefaults.standard.string(forKey: key), + saved.hasPrefix("OpenGlass-"), saved.count == 18 { return saved } + let name = "OpenGlass-" + UUID().uuidString.prefix(8) + UserDefaults.standard.set(name, forKey: key) + return name + }() + + var onImage: ((Data) -> Void)? + var onAudio: ((Data) -> Void)? + var onStatus: ((String) -> Void)? + var onFailure: ((String) -> Void)? + var onAudioConnectionChanged: ((Bool) -> Void)? + + private let queue = DispatchQueue(label: "org.opensqz.rokid.ingress") + private var listener: NWListener? + private var clients: [UUID: RokidIngressConnection] = [:] + private var audioClient: UUID? + private var imageCount = 0 + private var audioPacketCount = 0 + private var conversationPhase = "waiting" + private var completedTurns = 0 + private var speechDiagnostics = OpenGlassSpeechDiagnostics() + private var speechGate = "receiver_stopped" + private let appBuild = Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "unknown" + private var epoch = UUID() + // Read/written only on queue. epoch belongs to the main queue and only + // filters UI callbacks; this also prevents an old listener accepting work. + private var servingGeneration: UUID? + private var imageDeliveryInFlight = false + private var pendingAudioBytes = 0 + private let maximumPendingAudioBytes = 64_000 // Two seconds at 16 kHz PCM16. + + func start(port: UInt16 = 18080) throws { + dispatchPrecondition(condition: .onQueue(.main)) + guard port > 0 else { throw RokidWireError.invalid("Invalid listen port") } + let parameters = NWParameters.tcp + parameters.allowLocalEndpointReuse = true + let newListener = try NWListener(using: parameters, on: NWEndpoint.Port(rawValue: port)!) + newListener.service = NWListener.Service(name: Self.serviceName, type: Self.serviceType) + stop() + let generation = UUID() + epoch = generation + listener = newListener + queue.sync { servingGeneration = generation } + newListener.stateUpdateHandler = { [weak self, weak newListener] state in + guard let self else { return } + switch state { + case .ready: self.deliver(generation) { $0.onStatus?("正在接收 Rokid，端口 \(port)") } + case .failed(let error): + self.deliver(generation) { + let message = "接收服务失败：\(error.localizedDescription)" + $0.onStatus?(message) + $0.onFailure?(message) + } + newListener?.cancel() + default: break + } + } + newListener.newConnectionHandler = { [weak self] connection in + guard let self, self.servingGeneration == generation, self.clients.count < 6 else { + connection.cancel() + return + } + let client = RokidIngressConnection(connection: connection, queue: self.queue) + self.clients[client.id] = client + client.healthJSON = { [weak self] in + guard let self else { return "{}" } + return self.healthJSON() + } + client.onImage = { [weak self] data in + guard let self else { return } + self.imageCount += 1 + // A busy UI must not retain an unbounded queue of full JPEGs. + guard !self.imageDeliveryInFlight else { return } + self.imageDeliveryInFlight = true + self.deliver(generation, completionOnQueue: { [weak self] in + guard let self, self.servingGeneration == generation else { return } + self.imageDeliveryInFlight = false + }) { $0.onImage?(data) } + } + client.onAudioOpen = { [weak self, weak client] in + guard let self, let client else { return } + // A reconnect replaces the previous audio source instead of interleaving PCM. + let previous = self.audioClient + self.audioClient = client.id + if let previous { self.clients[previous]?.close() } + self.deliver(generation) { $0.onAudioConnectionChanged?(true) } + } + client.onAudio = { [weak self, weak client] data in + guard let self, let client, self.audioClient == client.id else { return } + guard self.pendingAudioBytes + data.count <= self.maximumPendingAudioBytes else { + // End the stream explicitly instead of silently introducing + // holes in a recognition request while the UI is blocked. + client.close() + self.deliver(generation) { $0.onStatus?("音频处理未及时跟上，正在等待 Rokid 重连。") } + return + } + self.audioPacketCount += 1 + self.pendingAudioBytes += data.count + self.deliver(generation, completionOnQueue: { [weak self] in + guard let self, self.servingGeneration == generation else { return } + self.pendingAudioBytes -= data.count + }) { $0.onAudio?(data) } + } + client.onClosed = { [weak self, weak client] in + guard let self, let client else { return } + self.clients.removeValue(forKey: client.id) + if self.audioClient == client.id { + self.audioClient = nil + self.deliver(generation) { $0.onAudioConnectionChanged?(false) } + } + } + client.start() + } + newListener.start(queue: queue) + } + + /// Only operational metadata is exposed; recognized speech and answers + /// never appear in network diagnostics. + func setConversationStatus(phase: String, completedTurns: Int, + asr: OpenGlassSpeechDiagnostics, gate: String) { + dispatchPrecondition(condition: .onQueue(.main)) + let safePhase = ["paused", "listening", "finishing", "cooldown", "thinking", "speaking", "waiting"].contains(phase) ? phase : "waiting" + let generation = epoch + queue.async { [weak self] in + guard let self, self.servingGeneration == generation else { return } + self.conversationPhase = safePhase + self.completedTurns = max(0, completedTurns) + self.speechDiagnostics = asr + self.speechGate = gate + } + } + + private func healthJSON() -> String { + let asr = speechDiagnostics + var details: [String: Any] = [ + "state": asr.state.rawValue, "gate": speechGate, + "sessions": asr.sessions, "accepted_samples": asr.acceptedSamples, + "session_samples": asr.sessionSamples, "last_rms": asr.lastRMS, + "session_peak_rms": asr.sessionPeakRMS, "partial_count": asr.partialCount, + "nonempty_partial_count": asr.nonemptyPartialCount, + "empty_final_count": asr.emptyFinalCount, "nonempty_final_count": asr.nonemptyFinalCount, + "failure_count": asr.failureCount, "last_failure_domain": asr.lastFailureDomain, + "last_end_reason": asr.lastEndReason + ] + if let code = asr.lastFailureCode { details["last_failure_code"] = code } + let status: [String: Any] = [ + "service": "openglass-ios", "protocol": 1, "receiver_name": Self.serviceName, + "app_build": appBuild, "image_count": imageCount, + "audio_packets_in": audioPacketCount, "audio_connected": audioClient != nil, + "conversation_phase": conversationPhase, "completed_turns": completedTurns, + "asr": details + ] + guard let data = try? JSONSerialization.data(withJSONObject: status), + let json = String(data: data, encoding: .utf8) else { return "{}" } + return json + } + + func stop() { + dispatchPrecondition(condition: .onQueue(.main)) + epoch = UUID() + listener?.cancel() + listener = nil + queue.sync { + servingGeneration = nil + let closing = Array(clients.values) + clients.removeAll() + audioClient = nil + imageCount = 0 + audioPacketCount = 0 + conversationPhase = "waiting" + completedTurns = 0 + speechDiagnostics.reset() + speechGate = "receiver_stopped" + imageDeliveryInFlight = false + pendingAudioBytes = 0 + closing.forEach { $0.close() } + } + onAudioConnectionChanged?(false) + } + + private func deliver(_ generation: UUID, completionOnQueue: (() -> Void)? = nil, + _ callback: @escaping (RokidIngressServer) -> Void) { + DispatchQueue.main.async { [weak self] in + guard let self else { return } + if self.epoch == generation { callback(self) } + if let completionOnQueue { self.queue.async(execute: completionOnQueue) } + } + } +} + +private final class RokidIngressConnection { + let id = UUID() + var onImage: ((Data) -> Void)? + var onAudio: ((Data) -> Void)? + var onAudioOpen: (() -> Void)? + var onClosed: (() -> Void)? + var healthJSON: (() -> String)? + private let connection: NWConnection + private let queue: DispatchQueue + private var buffer = Data() + private var head: RokidHTTPHead? + private var decoder = RokidWebSocketDecoder() + private var websocket = false + private var closed = false + private var responding = false + private var deadline: DispatchSourceTimer? + private var lastInput = Date() + private let created = Date() + + init(connection: NWConnection, queue: DispatchQueue) { + self.connection = connection + self.queue = queue + } + + func start() { + connection.stateUpdateHandler = { [weak self] state in + switch state { + case .failed, .cancelled: self?.close() + default: break + } + } + connection.start(queue: queue) + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now() + 5, repeating: 5) + timer.setEventHandler { [weak self] in + guard let self else { return } + let age = Date().timeIntervalSince(self.websocket ? self.lastInput : self.created) + if age > (self.websocket ? 30 : 15) { self.close() } + } + deadline = timer + timer.resume() + receive() + } + + func close() { + guard !closed else { return } + closed = true + deadline?.cancel() + deadline = nil + connection.cancel() + onClosed?() + } + + private func receive() { + guard !closed, !responding else { return } + connection.receive(minimumIncompleteLength: 1, maximumLength: 64 * 1024) { [weak self] data, _, done, error in + guard let self, !self.closed else { return } + if let data, !data.isEmpty { + self.lastInput = Date() + do { + if self.websocket { try self.consumeAudio(data) } + else { try self.consumeHTTP(data) } + } catch { + if self.websocket { self.closeWebSocket(code: 1002) } + else { self.respond(status: "400 Bad Request", body: "Invalid Rokid request") } + } + } + if error != nil || (done && !self.responding) { self.close() } + else { self.receive() } + } + } + + private func consumeHTTP(_ data: Data) throws { + buffer.append(data) + guard buffer.count <= RokidHTTPHead.maximumImageBytes + RokidHTTPHead.maximumHeaderBytes else { + throw RokidWireError.invalid("Image too large") + } + if head == nil { + guard let parsed = try RokidHTTPHead.parse(buffer) else { return } + head = parsed + buffer = Data(buffer.dropFirst(parsed.bytesConsumed)) + if parsed.method == "GET", parsed.path == "/health" { + respond(status: "200 OK", body: healthJSON?() ?? "{}", type: "application/json") + return + } + if parsed.path == "/rokid/audio" { + let accept = try parsed.webSocketAccept() + websocket = true + let reply = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: \(accept)\r\n\r\n" + connection.send(content: Data(reply.utf8), completion: .contentProcessed { [weak self] error in + if error != nil { self?.close() } + }) + onAudioOpen?() + let remainder = buffer + buffer.removeAll(keepingCapacity: false) + if !remainder.isEmpty { try consumeAudio(remainder) } + return + } + guard parsed.method == "POST", parsed.path == "/rokid/image" else { + respond(status: "404 Not Found", body: "Unknown route") + return + } + guard let length = parsed.contentLength, length > 3, length <= RokidHTTPHead.maximumImageBytes, + parsed.headers["content-type"]?.lowercased().split(separator: ";").first == "image/jpeg" else { + throw RokidWireError.invalid("Expected a bounded JPEG body") + } + if parsed.headers["expect"]?.lowercased() == "100-continue" { + connection.send(content: Data("HTTP/1.1 100 Continue\r\n\r\n".utf8), completion: .contentProcessed { _ in }) + } + } + guard let count = head?.contentLength, buffer.count >= count else { return } + let jpeg = Data(buffer.prefix(count)) + guard jpeg.starts(with: [0xff, 0xd8]), jpeg.suffix(2) == Data([0xff, 0xd9]) else { + throw RokidWireError.invalid("Invalid JPEG markers") + } + onImage?(jpeg) + respond(status: "200 OK", body: "{\"ok\":true}", type: "application/json") + } + + private func consumeAudio(_ data: Data) throws { + for event in try decoder.append(data) { + switch event { + case .binary(let pcm): + guard pcm.count % 2 == 0 else { throw RokidWireError.invalid("PCM16 requires complete samples") } + if !pcm.isEmpty { onAudio?(pcm) } + case .ping(let payload): + connection.send(content: RokidWebSocketDecoder.serverControlFrame(opcode: 10, payload: payload), + completion: .contentProcessed { [weak self] error in + if error != nil { self?.close() } + }) + case .close(let payload): + responding = true + connection.send(content: RokidWebSocketDecoder.serverControlFrame(opcode: 8, payload: payload), + completion: .contentProcessed { [weak self] _ in self?.close() }) + return + } + } + } + + private func closeWebSocket(code: UInt16) { + responding = true + let payload = Data([UInt8(code >> 8), UInt8(code & 0xff)]) + connection.send(content: RokidWebSocketDecoder.serverControlFrame(opcode: 8, payload: payload), + completion: .contentProcessed { [weak self] _ in self?.close() }) + } + + private func respond(status: String, body: String, type: String = "text/plain") { + guard !responding else { return } + responding = true + let bytes = Data(body.utf8) + var response = Data("HTTP/1.1 \(status)\r\nContent-Type: \(type)\r\nContent-Length: \(bytes.count)\r\nConnection: close\r\n\r\n".utf8) + response.append(bytes) + connection.send(content: response, completion: .contentProcessed { [weak self] _ in self?.close() }) + } +} diff --git a/ios/Sources/RokidWireProtocol.swift b/ios/Sources/RokidWireProtocol.swift new file mode 100644 index 0000000..17a2fe5 --- /dev/null +++ b/ios/Sources/RokidWireProtocol.swift @@ -0,0 +1,156 @@ +import Foundation +import CryptoKit + +enum RokidWireError: Error, LocalizedError { + case invalid(String) + var errorDescription: String? { + if case .invalid(let message) = self { return message } + return nil + } +} + +/// Only the two existing Rokid upload routes and a health probe are supported. +/// Parsing is incremental: TCP packet boundaries are not HTTP/WS message boundaries. +struct RokidHTTPHead: Equatable { + static let maximumHeaderBytes = 16 * 1024 + static let maximumImageBytes = 4 * 1024 * 1024 + let method: String + let path: String + let headers: [String: String] + let bytesConsumed: Int + + var contentLength: Int? { + guard let value = headers["content-length"], !value.isEmpty, + value.allSatisfy({ $0 >= "0" && $0 <= "9" }) else { return nil } + return Int(value) + } + + static func parse(_ data: Data) throws -> RokidHTTPHead? { + guard let boundary = data.range(of: Data("\r\n\r\n".utf8)) else { + if data.count > maximumHeaderBytes { throw RokidWireError.invalid("HTTP header too large") } + return nil + } + let count = data.distance(from: data.startIndex, to: boundary.upperBound) + guard count <= maximumHeaderBytes, + let text = String(data: data.prefix(count - 4), encoding: .ascii) else { + throw RokidWireError.invalid("Invalid HTTP header") + } + let lines = text.components(separatedBy: "\r\n") + let request = lines[0].split(separator: " ", omittingEmptySubsequences: false) + guard request.count == 3, request[2] == "HTTP/1.1" || request[2] == "HTTP/1.0" else { + throw RokidWireError.invalid("Invalid request line") + } + var headers: [String: String] = [:] + for line in lines.dropFirst() { + guard let colon = line.firstIndex(of: ":"), colon != line.startIndex, + line.first != " ", line.first != "\t" else { + throw RokidWireError.invalid("Invalid HTTP field") + } + let name = line[.. String { + let connection = (headers["connection"] ?? "").lowercased() + .split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) } + guard method == "GET", path == "/rokid/audio", + headers["upgrade"]?.lowercased() == "websocket", connection.contains("upgrade"), + headers["sec-websocket-version"] == "13", + let key = headers["sec-websocket-key"], Data(base64Encoded: key)?.count == 16, + (contentLength ?? 0) == 0 else { + throw RokidWireError.invalid("Invalid WebSocket upgrade") + } + let digest = Insecure.SHA1.hash(data: Data((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").utf8)) + return Data(digest).base64EncodedString() + } +} + +enum RokidWebSocketEvent { + case binary(Data) + case ping(Data) + case close(Data) +} + +struct RokidWebSocketDecoder { + static let maximumMessageBytes = 128 * 1024 + private var buffer = Data() + private var fragmented: Data? + + mutating func append(_ data: Data) throws -> [RokidWebSocketEvent] { + buffer.append(data) + var events: [RokidWebSocketEvent] = [] + while buffer.count >= 2 { + let bytes = [UInt8](buffer.prefix(14)) + let final = bytes[0] & 0x80 != 0 + let opcode = bytes[0] & 0x0f + guard bytes[0] & 0x70 == 0, bytes[1] & 0x80 != 0 else { + throw RokidWireError.invalid("Client frames must be masked; extensions are disabled") + } + var offset = 2 + var length = UInt64(bytes[1] & 0x7f) + if length == 126 { + guard buffer.count >= 4 else { break } + length = UInt64(bytes[2]) << 8 | UInt64(bytes[3]) + guard length >= 126 else { throw RokidWireError.invalid("Noncanonical frame length") } + offset = 4 + } else if length == 127 { + guard buffer.count >= 10 else { break } + guard bytes[2] & 0x80 == 0 else { throw RokidWireError.invalid("Invalid frame length") } + length = bytes[2..<10].reduce(UInt64(0)) { ($0 << 8) | UInt64($1) } + guard length >= 65536 else { throw RokidWireError.invalid("Noncanonical frame length") } + offset = 10 + } + guard length <= UInt64(Self.maximumMessageBytes) else { + throw RokidWireError.invalid("Audio frame too large") + } + guard [0, 2, 8, 9, 10].contains(Int(opcode)) else { + throw RokidWireError.invalid("Only binary PCM audio is supported") + } + if opcode >= 8 && (!final || length > 125) { + throw RokidWireError.invalid("Invalid WebSocket control frame") + } + let total = offset + 4 + Int(length) + guard buffer.count >= total else { break } + let mask = Array(buffer.dropFirst(offset).prefix(4)) + let payload = Data(buffer.dropFirst(offset + 4).prefix(Int(length)).enumerated() + .map { $0.element ^ mask[$0.offset % 4] }) + buffer = Data(buffer.dropFirst(total)) + switch opcode { + case 2: + guard fragmented == nil else { throw RokidWireError.invalid("Nested fragmented message") } + if final { events.append(.binary(payload)) } else { fragmented = payload } + case 0: + guard var message = fragmented else { throw RokidWireError.invalid("Unexpected continuation") } + guard message.count + payload.count <= Self.maximumMessageBytes else { + throw RokidWireError.invalid("Audio message too large") + } + message.append(payload) + if final { fragmented = nil; events.append(.binary(message)) } else { fragmented = message } + case 8: + guard payload.count != 1 else { throw RokidWireError.invalid("Invalid close payload") } + events.append(.close(payload)) + return events + case 9: events.append(.ping(payload)) + default: break // pong + } + } + guard buffer.count <= Self.maximumMessageBytes + 14 else { + throw RokidWireError.invalid("Receive buffer exceeded") + } + return events + } + + static func serverControlFrame(opcode: UInt8, payload: Data) -> Data { + precondition(payload.count <= 125) + var data = Data([0x80 | opcode, UInt8(payload.count)]) + data.append(payload) + return data + } +} diff --git a/ios/scripts/build_ipa.sh b/ios/scripts/build_ipa.sh new file mode 100644 index 0000000..c662da3 --- /dev/null +++ b/ios/scripts/build_ipa.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Build an unsigned device IPA for local signing on Windows. No Apple secrets. +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +UPSTREAM="${ROOT}/tmp/MiniCPM-V-Apps" +BUILD_ROOT="${ROOT}/tmp/ios-build" +ARTIFACTS="${ROOT}/artifacts" +BUILD_NUMBER="${GITHUB_RUN_NUMBER:-1}" +export ROOT UPSTREAM BUILD_ROOT ARTIFACTS + +if [[ "$(uname -s)" != Darwin ]]; then + echo "build_ipa.sh requires macOS with Xcode; run the iOS IPA GitHub Actions workflow from Windows." >&2 + exit 1 +fi +python3 "${ROOT}/ios/scripts/prepare_upstream.py" --build-number "${BUILD_NUMBER}" + +# Check every generated root before upstream's build script cleans its own dirs. +python3 - <<'PY' +import os +from pathlib import Path +root = Path(os.environ['ROOT']).resolve() +upstream = Path(os.environ['UPSTREAM']).resolve() +if not upstream.is_relative_to(root / 'tmp'): + raise SystemExit('Upstream build root escapes repository tmp/') +for relative in ('llama.cpp-omni/build-ios-sim', 'llama.cpp-omni/build-ios-device', + 'llama.cpp-omni/build-macos', 'llama.cpp-omni/build-apple', + 'MiniCPM-V-demo/thirdparty/llama.xcframework'): + if not (upstream / relative).resolve().is_relative_to(upstream): + raise SystemExit('Upstream build output resolves outside its checkout') +for name in ('BUILD_ROOT', 'ARTIFACTS'): + if not Path(os.environ[name]).resolve().is_relative_to(root): + raise SystemExit(name + ' escapes repository') +PY + +FRAMEWORK="${UPSTREAM}/MiniCPM-V-demo/thirdparty/llama.xcframework" +FRAMEWORK_STAMP="${UPSTREAM}/MiniCPM-V-demo/thirdparty/.openglass-framework-key" +FRAMEWORK_KEY="$(python3 - <<'PY' +import hashlib, os, subprocess +from pathlib import Path +payload = (Path(os.environ['ROOT']) / 'ios/upstream-lock.json').read_bytes() +payload += subprocess.check_output(['xcodebuild', '-version']) +payload += subprocess.check_output(['uname', '-m']) +print(hashlib.sha256(payload).hexdigest()) +PY +)" +if [[ ! -f "${FRAMEWORK}/Info.plist" || ! -f "${FRAMEWORK_STAMP}" || "$(cat "${FRAMEWORK_STAMP}")" != "${FRAMEWORK_KEY}" ]]; then + (cd "${UPSTREAM}" && MINIMAL_MODE=ios-device bash scripts/build_xcframework.sh) + printf '%s\n' "${FRAMEWORK_KEY}" > "${FRAMEWORK_STAMP}" +fi + +mkdir -p "${BUILD_ROOT}" "${ARTIFACTS}" +PROJECT="${UPSTREAM}/MiniCPM-V-demo.xcodeproj" +COMMON_ARGS=( + -project "${PROJECT}" -scheme MiniCPM-V-demo + -clonedSourcePackagesDirPath "${BUILD_ROOT}/SourcePackages" + -onlyUsePackageVersionsFromResolvedFile +) +xcodebuild "${COMMON_ARGS[@]}" -resolvePackageDependencies 2>&1 | tee "${BUILD_ROOT}/resolve.log" +xcodebuild "${COMMON_ARGS[@]}" \ + -configuration Release -sdk iphoneos -destination 'generic/platform=iOS' \ + -derivedDataPath "${BUILD_ROOT}/DerivedData" -disableAutomaticPackageResolution \ + CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY='' \ + DEVELOPMENT_TEAM='' CODE_SIGN_ENTITLEMENTS='' \ + build 2>&1 | tee "${BUILD_ROOT}/xcodebuild.log" + +APP="${BUILD_ROOT}/DerivedData/Build/Products/Release-iphoneos/MiniCPM-V-demo.app" +export APP +python3 - <<'PY' +import json, os, plistlib, subprocess +from pathlib import Path +root, app = Path(os.environ['ROOT']), Path(os.environ['APP']) +lock = json.loads((root / 'ios/upstream-lock.json').read_text()) +info = plistlib.loads((app / 'Info.plist').read_bytes()) +if info.get('CFBundleIdentifier') != lock['bundle_identifier']: + raise SystemExit('Built app has the wrong bundle identifier') +if info.get('CFBundleDisplayName') != lock['display_name']: + raise SystemExit('Built app has the wrong display name') +if info.get('CFBundleShortVersionString') != lock['version']: + raise SystemExit('Built app has the wrong version') +if (app / 'embedded.mobileprovision').exists(): + raise SystemExit('Unsigned build unexpectedly contains a provisioning profile') +if any(app.rglob('*.gguf')) or any(app.rglob('*.safetensors')): + raise SystemExit('Model weights must not be packaged into the IPA') +manifest = json.loads((Path(os.environ['UPSTREAM']) / '.openglass-prepared.json').read_text()) +manifest.update({'signed': False, 'xcode': subprocess.check_output(['xcodebuild', '-version'], text=True).strip(), + 'commit': subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip()}) +(Path(os.environ['ARTIFACTS']) / 'build-info.json').write_text(json.dumps(manifest, indent=2) + '\n') +PY + +# mktemp creates a fresh staging tree; no broad rm -rf or stale Payload merging. +STAGING="$(mktemp -d "${BUILD_ROOT}/package.XXXXXX")" +mkdir -p "${STAGING}/Payload" +ditto "${APP}" "${STAGING}/Payload/OpenGlass.app" +IPA="${ARTIFACTS}/OpenGlass-1.0.ipa" +ditto -c -k --keepParent "${STAGING}/Payload" "${STAGING}/OpenGlass-1.0.ipa" +mv -f "${STAGING}/OpenGlass-1.0.ipa" "${IPA}" +unzip -tq "${IPA}" +echo "Unsigned IPA ready: ${IPA}" +echo "Sign this IPA on your Windows computer before installing on iPhone." diff --git a/ios/scripts/prepare_upstream.py b/ios/scripts/prepare_upstream.py new file mode 100644 index 0000000..f3d0749 --- /dev/null +++ b/ios/scripts/prepare_upstream.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Prepare a pinned, disposable iOS build tree. Requires Python 3.10+ and Git. + +Only files below this repository's tmp/ directory may be generated. Each run +derives patched upstream files from the locked Git object, so it is idempotent +and never accumulates edits. The upstream model download UI is left intact. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import plistlib +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +APP = "MiniCPM-V-demo" +PROJECT = f"{APP}.xcodeproj/project.pbxproj" +PACKAGES = f"{APP}.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved" +APP_CONFIG_IDS = ("FA3FA2782C1176E90077F732", "FA3FA2792C1176E90077F732") +LOCAL_NETWORK_DESCRIPTION = "接收同一局域网内 Rokid 眼镜的图像和麦克风音频。" +SPEECH_DESCRIPTION = "在 iPhone 上将眼镜麦克风语音转换为文字；仅使用设备端识别。" + + +class PrepareError(RuntimeError): + pass + + +def git(directory: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(directory), *args], check=True, + stdout=subprocess.PIPE, encoding="utf-8", errors="strict", + ) + return result.stdout.strip() + + +def git_file(directory: Path, revision: str, path: str) -> bytes: + return subprocess.check_output(["git", "-C", str(directory), "show", f"{revision}:{path}"]) + + +def safe_generated_path(root: Path, path: Path) -> Path: + """Reject the repository itself, outside directories, and escaping symlinks.""" + root = root.resolve() + temporary = (root / "tmp").resolve() + if temporary != root / "tmp": + raise PrepareError("tmp/ must be a real directory inside the repository") + resolved = path.resolve() + if resolved == temporary or not resolved.is_relative_to(temporary): + raise PrepareError(f"Generated path must be strictly below {temporary}: {path}") + return resolved + + +def read_lock(root: Path) -> dict: + lock = json.loads((root / "ios/upstream-lock.json").read_text(encoding="utf-8")) + for revision in (lock["commit"], lock["submodule"]["commit"], *lock["swift_packages"].values()): + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise PrepareError("Upstream dependencies must use full 40-character Git SHAs") + if lock["submodule"]["path"] != "llama.cpp-omni": + raise PrepareError("Unexpected inference submodule path") + return lock + + +def ensure_checkout(destination: Path, lock: dict, skip_submodules: bool) -> None: + if not destination.exists(): + destination.mkdir(parents=True) + if not (destination / ".git").exists(): + if any(destination.iterdir()): + raise PrepareError("Refusing to initialize a nonempty, non-Git destination") + git(destination, "init") + git(destination, "remote", "add", "origin", lock["repository"]) + git(destination, "fetch", "--depth", "1", "origin", lock["commit"]) + git(destination, "checkout", "--detach", "FETCH_HEAD") + if Path(git(destination, "rev-parse", "--show-toplevel")).resolve() != destination.resolve(): + raise PrepareError("Destination is not the root of its own Git checkout") + if git(destination, "remote", "get-url", "origin").rstrip("/") != lock["repository"]: + raise PrepareError("Existing checkout has a different origin; choose a fresh tmp/ directory") + if git(destination, "rev-parse", "HEAD") != lock["commit"]: + raise PrepareError("Existing checkout is not at the locked commit; no reset was performed") + managed_files = {PROJECT, PACKAGES, f"{APP}/SceneDelegate.swift", f"{APP}/Info.plist", f"{APP}/{APP}.entitlements"} + changed = set(git(destination, "diff", "HEAD", "--name-only", "--ignore-submodules=all").splitlines()) + unexpected = changed - managed_files + if unexpected: + raise PrepareError("Upstream has edits outside generated files: " + ", ".join(sorted(unexpected))) + recorded = git(destination, "ls-tree", "HEAD", lock["submodule"]["path"]).split() + if len(recorded) != 4 or recorded[0] != "160000" or recorded[2] != lock["submodule"]["commit"]: + raise PrepareError("Upstream Git tree does not match the locked inference submodule") + if not skip_submodules: + git(destination, "submodule", "update", "--init", "--recursive", "--depth", "1", "--checkout") + submodule = destination / lock["submodule"]["path"] + if git(submodule, "rev-parse", "HEAD") != lock["submodule"]["commit"]: + raise PrepareError("Inference submodule checkout does not match its lock") + if git(submodule, "status", "--porcelain", "--untracked-files=no"): + raise PrepareError("Inference submodule has local edits; refusing a non-reproducible build") + + +def replace_once(text: str, old: str, new: str) -> str: + if text.count(old) != 1: + raise PrepareError(f"Expected exactly one upstream patch anchor: {old[:100]!r}") + return text.replace(old, new, 1) + + +def object_id(kind: str, name: str) -> str: + return hashlib.sha256(f"OpenGlass:{kind}:{name}".encode()).hexdigest()[:24].upper() + + +def patch_project(original: str, source_names: list[str], lock: dict, build_number: str) -> str: + if not re.fullmatch(r"[1-9][0-9]{0,8}(?:\.[0-9]{1,3}){0,2}", build_number): + raise PrepareError("Build number must be a positive Apple-compatible numeric version") + if not source_names or len(set(source_names)) != len(source_names): + raise PrepareError("Expected a nonempty list of distinct Swift source files") + if any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_+.\-]*\.swift", name) for name in source_names): + raise PrepareError("Swift source filenames must be safe PBX filenames") + source_names = sorted(source_names) + group_id = object_id("group", "OpenGlass") + build_files, references, children, sources = [], [], [], [] + for name in source_names: + reference, build = object_id("file", name), object_id("build", name) + if reference in original or build in original: + raise PrepareError("Generated PBX identifier collides with an upstream object") + build_files.append(f"\t\t{build} /* {name} in Sources */ = {{isa = PBXBuildFile; fileRef = {reference} /* {name} */; }};\n") + references.append(f'\t\t{reference} /* {name} */ = {{isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "{name}"; sourceTree = ""; }};\n') + children.append(f"\t\t\t\t{reference} /* {name} */,\n") + sources.append(f"\t\t\t\t{build} /* {name} in Sources */,\n") + text = replace_once(original, "/* Begin PBXBuildFile section */\n", "/* Begin PBXBuildFile section */\n" + "".join(build_files)) + text = replace_once(text, "/* Begin PBXFileReference section */\n", "/* Begin PBXFileReference section */\n" + "".join(references)) + group = (f"\t\t{group_id} /* OpenGlass */ = {{\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n" + + "".join(children) + f'\t\t\t);\n\t\t\tname = OpenGlass;\n\t\t\tpath = "{APP}/OpenGlass";\n\t\t\tsourceTree = SOURCE_ROOT;\n\t\t}};\n') + text = replace_once(text, "/* Begin PBXGroup section */\n", "/* Begin PBXGroup section */\n" + group) + root_group = "\t\tFA3FA25A2C1176E80077F732 = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n" + text = replace_once(text, root_group, root_group + f"\t\t\t\t{group_id} /* OpenGlass */,\n") + phase = "\t\tFA3FA25F2C1176E80077F732 /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n" + text = replace_once(text, phase, phase + "".join(sources)) + settings = { + "PRODUCT_BUNDLE_IDENTIFIER": lock["bundle_identifier"], + "INFOPLIST_KEY_CFBundleDisplayName": json.dumps(lock["display_name"]), + "INFOPLIST_KEY_NSLocalNetworkUsageDescription": json.dumps(LOCAL_NETWORK_DESCRIPTION, ensure_ascii=False), + "INFOPLIST_KEY_NSSpeechRecognitionUsageDescription": json.dumps(SPEECH_DESCRIPTION, ensure_ascii=False), + "MARKETING_VERSION": lock["version"], + "CURRENT_PROJECT_VERSION": build_number, + } + # Patch only the app's two configurations, retaining upstream test settings. + for config_id in APP_CONFIG_IDS: + pattern = rf"(?ms)^\t\t{config_id} /\* .*? \*/ = \{{\n.*?^\t\t\}};" + match = re.search(pattern, text) + if match is None: + raise PrepareError(f"Missing app build configuration {config_id}") + config = match.group(0) + for key, value in settings.items(): + config, count = re.subn(rf"(?m)^(\t{{4}}{key} = )[^\n;]*;", lambda m: f"{m[1]}{value};", config) + if count != 1: + raise PrepareError(f"Expected one {key} in app build configuration") + text = text[:match.start()] + config + text[match.end():] + return text + + +def write_generated(root: Path, destination: Path, relative: str, data: bytes) -> None: + path = safe_generated_path(root, destination / relative) + if not path.is_relative_to(destination): + raise PrepareError("Generated file escapes its upstream checkout") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +def prepare(root: Path, destination: Path, lock: dict, build_number: str) -> dict: + destination = safe_generated_path(root, destination) + source_dir = root / "ios/Sources" + sources = sorted(source_dir.glob("*.swift")) + for source in sources: + if source.is_symlink() or source.resolve().parent != source_dir.resolve(): + raise PrepareError("Overlay sources must be regular files in ios/Sources") + revision = lock["commit"] + original = git_file(destination, revision, PROJECT).decode("utf-8").replace("\r\n", "\n") + project = patch_project(original, [source.name for source in sources], lock, build_number) + scene = git_file(destination, revision, f"{APP}/SceneDelegate.swift").decode("utf-8").replace("\r\n", "\n") + scene = replace_once(scene, "let vc = MBHomeViewController()", "let vc = OpenGlassViewController()") + info = plistlib.loads(git_file(destination, revision, f"{APP}/Info.plist")) + info.update({"CFBundleDisplayName": lock["display_name"], + "NSLocalNetworkUsageDescription": LOCAL_NETWORK_DESCRIPTION, + "NSBonjourServices": ["_openglass._tcp"], + "NSSpeechRecognitionUsageDescription": SPEECH_DESCRIPTION}) + # No background-mode claim: receiving and inference require foreground use. + info.setdefault("NSAppTransportSecurity", {})["NSAllowsLocalNetworking"] = True + packages_data = git_file(destination, revision, PACKAGES) + pins = {item["identity"]: item["state"]["revision"] for item in json.loads(packages_data)["pins"]} + if pins != lock["swift_packages"]: + raise PrepareError("Package.resolved dependencies differ from upstream-lock.json") + outputs = {PROJECT: project.encode("utf-8"), f"{APP}/SceneDelegate.swift": scene.encode("utf-8"), + f"{APP}/Info.plist": plistlib.dumps(info, sort_keys=False), PACKAGES: packages_data, + # Free personal-team signing cannot use increased-memory-limit. + f"{APP}/{APP}.entitlements": plistlib.dumps({})} + outputs.update({f"{APP}/OpenGlass/{source.name}": source.read_bytes() for source in sources}) + for relative, content in outputs.items(): + write_generated(root, destination, relative, content) + manifest = {"upstream_commit": revision, "submodule_commit": lock["submodule"]["commit"], + "bundle_identifier": lock["bundle_identifier"], "version": lock["version"], + "build_number": build_number, "sources": {source.name: hashlib.sha256(source.read_bytes()).hexdigest() for source in sources}} + write_generated(root, destination, ".openglass-prepared.json", (json.dumps(manifest, indent=2) + "\n").encode()) + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream-dir", type=Path, default=ROOT / "tmp/MiniCPM-V-Apps") + parser.add_argument("--build-number", default=os.environ.get("GITHUB_RUN_NUMBER", "1")) + parser.add_argument("--skip-submodules", action="store_true", help="Prepare sources only; cannot build the IPA yet") + args = parser.parse_args() + try: + destination = safe_generated_path(ROOT, args.upstream_dir) + lock = read_lock(ROOT) + ensure_checkout(destination, lock, args.skip_submodules) + manifest = prepare(ROOT, destination, lock, args.build_number) + except (PrepareError, subprocess.CalledProcessError, OSError, ValueError) as exc: + print(f"prepare_upstream: {exc}", file=sys.stderr) + return 1 + print(f"Prepared OpenGlass {manifest['version']} ({manifest['build_number']}), {len(manifest['sources'])} Swift sources in {destination}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ios/tests/main.swift b/ios/tests/main.swift new file mode 100644 index 0000000..4083a4b --- /dev/null +++ b/ios/tests/main.swift @@ -0,0 +1,85 @@ +import Foundation + +var checks = 0 +func check(_ condition: @autoclosure () throws -> Bool, _ label: String) { + do { guard try condition() else { fatalError("FAIL: \(label)") } } + catch { fatalError("FAIL: \(label): \(error)") } + checks += 1 +} +func rejects(_ label: String, _ action: () throws -> Void) { + do { try action(); fatalError("FAIL: expected rejection: \(label)") } + catch { checks += 1 } +} +func frame(_ payload: Data, opcode: UInt8 = 2, final: Bool = true) -> Data { + var data = Data([(final ? 0x80 : 0) | opcode]) + if payload.count < 126 { data.append(0x80 | UInt8(payload.count)) } + else if payload.count <= 65535 { + data.append(contentsOf: [0xfe, UInt8(payload.count >> 8), UInt8(payload.count & 0xff)]) + } else { + data.append(0xff) + for shift in stride(from: 56, through: 0, by: -8) { + data.append(UInt8((UInt64(payload.count) >> shift) & 0xff)) + } + } + let mask: [UInt8] = [0x37, 0xfa, 0x21, 0x3d] + data.append(contentsOf: mask) + data.append(contentsOf: payload.enumerated().map { $0.element ^ mask[$0.offset % 4] }) + return data +} +func binary(_ events: [RokidWebSocketEvent]) -> [Data] { + events.compactMap { if case .binary(let data) = $0 { return data }; return nil } +} + +let jpegHeader = Data("POST /rokid/image HTTP/1.1\r\nHost: phone\r\nContent-Type: image/jpeg\r\nContent-Length: 1280\r\n\r\n".utf8) +for split in 0.. Bool, _ label: String) { + guard condition() else { fatalError("FAIL: \(label)") } + checks += 1 +} +func checkTime(_ actual: TimeInterval, _ expected: TimeInterval, _ label: String) { + check(abs(actual - expected) < 0.000_000_001, label) +} + +// Startup is opt-in, including when a late callback arrives before any start. +var startup = OpenGlassVoiceLoopPolicy() +check(!startup.wantsListening && !startup.canListen(at: 100), "startup waits for user intent") +startup.retry(now: 100) +startup.receivedUtterance() +check(!startup.canListen(at: 200), "callbacks cannot create startup intent") +startup.requestListening() +check(startup.canListen(at: 200), "explicit start opens the gate") + +// Recognition failures keep a positive, bounded delay. Wakeups at the +// deadline may retry; repeated audio/frame notifications before it may not. +var silence = OpenGlassVoiceLoopPolicy() +silence.requestListening() +let expectedDelays: [TimeInterval] = [0.5, 1, 2, 4, 8, 8, 8] +var now: TimeInterval = 1_000 +for (index, expected) in expectedDelays.enumerated() { + let delay = silence.retry(now: now) + checkTime(delay, expected, "recognition failure retry \(index + 1) uses bounded backoff") + checkTime(silence.notBefore, now + expected, "retry sets a future deadline") + check(!silence.canListen(at: now), "retry cannot spin immediately") + check(!silence.canListen(at: silence.notBefore - 0.001), "retry waits through its deadline") + check(silence.canListen(at: silence.notBefore), "retry is allowed at its deadline") + now = silence.notBefore +} +for _ in 0..<1_000 { + checkTime(silence.retry(now: now), 8, "repeated recognition failures stay capped at eight seconds") + check(!silence.canListen(at: now), "capped retries still wait") + now = silence.notBefore +} +check(silence.retryCount == 5, "retry count saturates instead of growing without bound") +silence.receivedUtterance() +check(silence.retryCount == 0 && silence.canListen(at: now), "a recognized question clears backoff") +checkTime(silence.retry(now: now), 0.5, "a later failure starts with the short delay") + +// A retry timer and an echo hold are independent: whichever expires later wins. +var timing = OpenGlassVoiceLoopPolicy() +timing.requestListening() +timing.protectFromEcho(now: 10) +checkTime(timing.retry(now: 10), 0.5, "retry reports its delay independently of echo") +checkTime(timing.notBefore, 10.7, "short retry cannot shorten echo hold") +timing.receivedUtterance() +checkTime(timing.notBefore, 10.7, "successful recognition cannot clear echo hold") +timing.requestListening() +check(!timing.canListen(at: 10.6), "explicit listening request still respects echo") +check(timing.canListen(at: timing.notBefore), "echo hold opens at the boundary") +timing.retry(now: 20) +timing.retry(now: 20) +timing.protectFromEcho(now: 20) +checkTime(timing.notBefore, 21, "longer retry survives a shorter echo hold") +timing.protectFromEcho(now: 20.5) +checkTime(timing.notBefore, 21.2, "a later echo hold extends the gate") +timing.protectFromEcho(now: 20.6, delay: 0.1) +checkTime(timing.notBefore, 21.2, "a stale shorter hold cannot move the deadline backwards") +check(!timing.canListen(at: 20.7), "the first echo wakeup cannot bypass an extended hold") + +// All ingress and completion paths consult the same gate. These arrivals model +// a reconnect, new frame, PCM packet and an old timer while TTS echo can remain. +var arrivals = OpenGlassVoiceLoopPolicy() +arrivals.requestListening() +arrivals.protectFromEcho(now: 30) +for arrival in [30.0, 30.1, 30.4, 30.699] { + check(!arrivals.canListen(at: arrival), "an arrival cannot bypass the common echo gate") +} +arrivals.requestListening() +check(!arrivals.canListen(at: 30.699), "a manual restart during echo also waits") +check(arrivals.canListen(at: arrivals.notBefore), "arrivals become eligible after echo expires") + +// A user pause wins over already queued retry, successful-final and TTS events. +var stopped = OpenGlassVoiceLoopPolicy() +stopped.requestListening() +stopped.retry(now: 40) +stopped.protectFromEcho(now: 40) +stopped.block(reason: "temporary test state") +stopped.pause() +check(!stopped.wantsListening && stopped.blockedReason == nil && stopped.retryCount == 0, + "pause clears failure state while disabling intent") +checkTime(stopped.notBefore, 40.7, "pause preserves the acoustic hold") +stopped.retry(now: 41) +stopped.receivedUtterance() +stopped.protectFromEcho(now: 42) +check(!stopped.canListen(at: 1_000), "late callbacks cannot resurrect a stopped loop") +stopped.requestListening() +check(!stopped.canListen(at: 42.5), "resuming a paused loop still waits for recent echo") +check(stopped.canListen(at: stopped.notBefore), "only explicit resume restores eligibility") + +// Permanent errors preserve user intent but require an explicit request to +// retry. Neither traffic, a successful old utterance nor elapsed time clears it. +var blocked = OpenGlassVoiceLoopPolicy() +blocked.requestListening() +blocked.block(reason: "offline language unavailable") +check(blocked.wantsListening, "blocking preserves intended listening") +blocked.retry(now: 50) +blocked.receivedUtterance() +blocked.protectFromEcho(now: 50) +check(blocked.blockedReason == "offline language unavailable", "late success keeps the permanent block") +check(!blocked.canListen(at: 1_000), "permanent failure does not expire with time") +blocked.requestListening() +check(blocked.blockedReason == nil && blocked.retryCount == 0, "explicit request clears the block") +check(!blocked.canListen(at: 50.6), "clearing a permanent block cannot bypass echo") +check(blocked.canListen(at: blocked.notBefore), "cleared block allows listening after echo") +blocked.pause() +blocked.block(reason: "late permission callback") +blocked.receivedUtterance() +check(!blocked.wantsListening && !blocked.canListen(at: 1_000), "a late permanent error cannot enable intent") + +// Intent survives only when the caller explicitly restores it after background +// cleanup. The policy never enables itself merely because time has passed. +var foreground = OpenGlassVoiceLoopPolicy() +foreground.requestListening() +let wasListening = foreground.wantsListening +foreground.protectFromEcho(now: 60) +foreground.pause() +check(!foreground.canListen(at: 60.2), "background cleanup closes the gate") +if wasListening { foreground.requestListening() } +check(!foreground.canListen(at: 60.2), "foreground restoration preserves the echo deadline") +check(foreground.canListen(at: 61), "restored intent can resume after the deadline") +foreground.pause() +let wasPaused = foreground.wantsListening +if wasPaused { foreground.requestListening() } +check(!foreground.canListen(at: 1_000), "a previously paused loop stays paused on foreground") + +// Steady environmental noise can keep the RMS voice marker fresh. A stable +// transcript still finishes promptly instead of waiting for the 25-second cap. +check(OpenGlassUtteranceTiming.action(now: 4.499, startedAt: 0, + lastPacketAt: 4.499, lastVoiceAt: 4.499, + lastTranscriptAt: 2, hasText: true) == .listen, + "stable text waits the full 2.5 seconds") +check(OpenGlassUtteranceTiming.action(now: 4.5, startedAt: 0, + lastPacketAt: 4.5, lastVoiceAt: 4.5, + lastTranscriptAt: 2, hasText: true) == .finish, + "stable text finishes despite continuous RMS noise") +check(OpenGlassUtteranceTiming.action(now: 4.5, startedAt: 0, + lastPacketAt: 4.5, lastVoiceAt: 4.5, + lastTranscriptAt: 4, hasText: true) == .listen, + "a changed partial transcript extends the text deadline") +check(OpenGlassUtteranceTiming.action(now: 6.5, startedAt: 0, + lastPacketAt: 6.5, lastVoiceAt: 6.5, + lastTranscriptAt: 4, hasText: true) == .finish, + "the updated transcript finishes when it becomes stable") + +// An empty transcript must not reuse the stable-text deadline from old content. +check(OpenGlassUtteranceTiming.action(now: 10, startedAt: 0, + lastPacketAt: 10, lastVoiceAt: 10, + lastTranscriptAt: 0, hasText: false) == .listen, + "old transcript time cannot finalize current empty recognition") +check(OpenGlassUtteranceTiming.action(now: 10, startedAt: 0, + lastPacketAt: 10, lastVoiceAt: 10, + lastTranscriptAt: nil, hasText: true) == .listen, + "text without a stability timestamp does not finish prematurely") +check(OpenGlassUtteranceTiming.action(now: 24.999, startedAt: 0, + lastPacketAt: 24.999, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .listen, + "silent packets keep empty recognition alive before the duration cap") +check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, + lastPacketAt: 25, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .finish, + "continuous silence finishes at 25 seconds") +check(OpenGlassUtteranceTiming.action(now: 1.299, startedAt: 0, + lastPacketAt: 1.299, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .listen, + "voice silence without text does not end an utterance early") +check(OpenGlassUtteranceTiming.action(now: 1.3, startedAt: 0, + lastPacketAt: 1.3, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .listen, + "empty recognition survives the 1.3-second voice silence boundary") +check(OpenGlassUtteranceTiming.action(now: 24.999, startedAt: 0, + lastPacketAt: 24.999, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .listen, + "an old RMS voice marker cannot end empty recognition before its cap") +check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, + lastPacketAt: 25, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .finish, + "empty recognition with an old voice marker still reaches its cap") +check(OpenGlassUtteranceTiming.action(now: 1.299, startedAt: 0, + lastPacketAt: 1.299, lastVoiceAt: 0, + lastTranscriptAt: 0.5, hasText: true) == .listen, + "recognized text waits the full 1.3 seconds of voice silence") +check(OpenGlassUtteranceTiming.action(now: 1.3, startedAt: 0, + lastPacketAt: 1.3, lastVoiceAt: 0, + lastTranscriptAt: 0.5, hasText: true) == .finish, + "recognized text finishes after 1.3 seconds of voice silence") + +// A stopped transport times out earlier than an otherwise empty utterance. +check(OpenGlassUtteranceTiming.action(now: 4.999, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .listen, + "packet gap waits the full five seconds") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .audioTimeout, + "missing packets time out at five seconds") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: 5, + lastTranscriptAt: 0, hasText: true) == .finish, + "stable transcript finalization takes precedence over a packet timeout") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .audioTimeout, + "an empty transcript with an old voice marker still detects missing packets") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: true) == .finish, + "recognized text can finalize voice silence before a packet timeout") +check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .finish, + "duration cap finalization takes precedence over a packet timeout") + +// Diagnostics record only accepted samples and callback outcomes. An idle or +// finishing session cannot claim that late audio reached the recognizer. +let checksBeforeDiagnostics = checks +var diagnostics = OpenGlassSpeechDiagnostics() +check(diagnostics.state == .idle && diagnostics.sessions == 0, "diagnostics start idle") +check(diagnostics.lastEndReason == "not_started" && diagnostics.lastFailureCode == nil, + "initial diagnostics have no failure or end event") +diagnostics.appended(samples: 640, rms: 0.13) +diagnostics.partial(nonempty: true) +diagnostics.finishing(reason: "late_finish") +diagnostics.completed(nonempty: false, reason: "late_final") +check(diagnostics.acceptedSamples == 0 && diagnostics.sessionSamples == 0, + "idle audio is not counted as accepted") +check(diagnostics.partialCount == 0 && diagnostics.emptyFinalCount == 0, + "idle callbacks are not recognition results") +check(diagnostics.state == .idle && diagnostics.lastEndReason == "not_started", + "late callbacks cannot create a session") + +diagnostics.started() +check(diagnostics.state == .listening && diagnostics.sessions == 1, + "starting records one listening session") +check(diagnostics.lastEndReason == "none", "starting clears the previous end reason") +diagnostics.appended(samples: 640, rms: 0.13) +diagnostics.appended(samples: 320, rms: 0.05) +check(diagnostics.acceptedSamples == 960 && diagnostics.sessionSamples == 960, + "accepted sample totals count each appended buffer") +checkTime(diagnostics.lastRMS, 0.05, "last RMS follows the latest accepted buffer") +checkTime(diagnostics.sessionPeakRMS, 0.13, "session peak retains the loudest buffer") +diagnostics.appended(samples: 0, rms: 1) +diagnostics.appended(samples: -1, rms: 1) +check(diagnostics.acceptedSamples == 960 && diagnostics.sessionSamples == 960, + "empty and invalid sample counts are ignored") +checkTime(diagnostics.lastRMS, 0.05, "ignored buffers do not change the last RMS") +diagnostics.partial(nonempty: false) +diagnostics.partial(nonempty: true) +check(diagnostics.partialCount == 2 && diagnostics.nonemptyPartialCount == 1, + "partial diagnostics distinguish empty callbacks") +diagnostics.finishing(reason: "silence") +check(diagnostics.state == .finishing && diagnostics.lastEndReason == "silence", + "finishing records the utterance boundary") +diagnostics.finishing(reason: "duplicate_finish") +check(diagnostics.lastEndReason == "silence", "duplicate finishing preserves the original reason") +diagnostics.appended(samples: 640, rms: 0.9) +check(diagnostics.acceptedSamples == 960 && diagnostics.sessionSamples == 960, + "audio after endAudio is not counted as accepted") +checkTime(diagnostics.lastRMS, 0.05, "finishing ignores late RMS values") +diagnostics.partial(nonempty: true) +check(diagnostics.partialCount == 3 && diagnostics.nonemptyPartialCount == 2, + "recognition may report a partial while finishing") +diagnostics.completed(nonempty: false, reason: "empty_final") +check(diagnostics.state == .idle && diagnostics.emptyFinalCount == 1, + "empty final completes one active session") +check(diagnostics.lastEndReason == "empty_final", "final records its end reason") +diagnostics.completed(nonempty: false, reason: "duplicate_final") +diagnostics.completed(nonempty: true, reason: "stale_final") +check(diagnostics.emptyFinalCount == 1 && diagnostics.nonemptyFinalCount == 0, + "duplicate and stale finals cannot count a session twice") +check(diagnostics.lastEndReason == "empty_final", "late final cannot overwrite the completed reason") + +// Invalid RMS values are sanitized independently of the accepted sample count. +diagnostics.started() +check(diagnostics.sessions == 2 && diagnostics.sessionSamples == 0 && diagnostics.acceptedSamples == 960, + "a new session resets its sample count but preserves the cumulative total") +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "a new session resets both RMS measurements") +check(diagnostics.partialCount == 3 && diagnostics.emptyFinalCount == 1, + "callback totals survive session boundaries") +diagnostics.appended(samples: 10, rms: .nan) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "NaN RMS cannot poison diagnostic values") +diagnostics.appended(samples: 10, rms: .infinity) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "infinite RMS cannot poison diagnostic values") +diagnostics.appended(samples: 10, rms: -0.2) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "negative RMS is clamped to zero") +diagnostics.appended(samples: 10, rms: 0.2) +diagnostics.appended(samples: 10, rms: -.infinity) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0.2, + "invalid later RMS cannot erase the finite peak") +check(diagnostics.sessionSamples == 50 && diagnostics.acceptedSamples == 1_010, + "invalid measurements do not discard accepted sample counts") +diagnostics.completed(nonempty: true, reason: "recognized") +check(diagnostics.state == .idle && diagnostics.nonemptyFinalCount == 1 && diagnostics.emptyFinalCount == 1, + "nonempty final can complete directly from listening") + +// Synchronous start failures are visible even when no task became active. +diagnostics.failed(domain: "test.recognizer", code: 42, reason: "start_failed") +check(diagnostics.state == .idle && diagnostics.failureCount == 1 && diagnostics.sessions == 2, + "a synchronous failure counts without inventing a session") +check(diagnostics.lastFailureDomain == "test.recognizer" && diagnostics.lastFailureCode == 42 && + diagnostics.lastEndReason == "start_failed", "failure records only domain, code and reason") +diagnostics.started() +check(diagnostics.failureCount == 1 && diagnostics.lastFailureDomain == "test.recognizer" && + diagnostics.lastFailureCode == 42, "starting preserves the most recent failure metadata") +check(diagnostics.sessions == 3 && diagnostics.lastEndReason == "none", + "retry starts another session with no end reason") +diagnostics.appended(samples: 100, rms: 0.4) +diagnostics.failed(domain: "test.runtime", code: -7, reason: "recognition_failed") +check(diagnostics.state == .idle && diagnostics.failureCount == 2 && diagnostics.sessionSamples == 100, + "asynchronous failure preserves accepted samples for that session") +check(diagnostics.lastFailureDomain == "test.runtime" && diagnostics.lastFailureCode == -7, + "a newer failure replaces previous error metadata") +diagnostics.completed(nonempty: true, reason: "late_failure_final") +check(diagnostics.nonemptyFinalCount == 1 && diagnostics.lastEndReason == "recognition_failed", + "a final arriving after failure is ignored") + +diagnostics.started() +diagnostics.appended(samples: 200, rms: 0.1) +diagnostics.partial(nonempty: true) +diagnostics.stopped(reason: "audio_disconnected") +check(diagnostics.state == .idle && diagnostics.lastEndReason == "audio_disconnected", + "explicit stop records its reason") +check(diagnostics.sessions == 4 && diagnostics.sessionSamples == 200 && diagnostics.acceptedSamples == 1_310, + "explicit stop preserves sample and session totals") +check(diagnostics.partialCount == 4 && diagnostics.nonemptyPartialCount == 3 && + diagnostics.failureCount == 2, "stopping does not change callback totals") +diagnostics.completed(nonempty: true, reason: "late_stop_final") +diagnostics.appended(samples: 640, rms: 1) +diagnostics.partial(nonempty: true) +check(diagnostics.nonemptyFinalCount == 1 && diagnostics.emptyFinalCount == 1 && + diagnostics.acceptedSamples == 1_310 && diagnostics.partialCount == 4, + "stop rejects late final, partial and audio callbacks") +check(diagnostics.lastEndReason == "audio_disconnected", "late callbacks preserve the stop reason") +diagnostics.stopped(reason: "user_pause") +check(diagnostics.lastEndReason == "user_pause" && diagnostics.sessions == 4, + "an idle explicit stop can update its reason without changing counts") + +diagnostics.reset() +check(diagnostics.state == .idle && diagnostics.sessions == 0 && diagnostics.acceptedSamples == 0 && + diagnostics.sessionSamples == 0, "reset clears state and sample totals") +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, "reset clears RMS values") +check(diagnostics.partialCount == 0 && diagnostics.nonemptyPartialCount == 0 && + diagnostics.emptyFinalCount == 0 && diagnostics.nonemptyFinalCount == 0 && diagnostics.failureCount == 0, + "reset clears every callback counter") +check(diagnostics.lastFailureDomain.isEmpty && diagnostics.lastFailureCode == nil && + diagnostics.lastEndReason == "not_started", "reset clears failure metadata and the end reason") +print("Passed \(checks - checksBeforeDiagnostics) OpenGlass speech-diagnostics checks") + +// Only audio that arrives after endAudio can be handed to the next request. +// A normal nonempty result, explicit stop or error discards the handoff. +let checksBeforeHandoff = checks +let handoffNow: TimeInterval = 10 +let firstPCM = Data([0, 1, 2, 3]) +let secondPCM = Data([4, 5, 6, 7]) +var handoff = OpenGlassSpeechHandoffBuffer(capacity: 8) +check(handoff.phase == .idle && handoff.byteCount == 0 && !handoff.hasReadyAudio(at: handoffNow), + "handoff begins idle and empty") +handoff.append(firstPCM) +check(handoff.byteCount == 0 && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "audio before finishing cannot enter the handoff") +handoff.begin() +check(handoff.phase == .collecting && !handoff.hasReadyAudio(at: handoffNow), + "begin creates an empty collecting phase") +handoff.append(firstPCM) +check(handoff.byteCount == 4 && !handoff.hasReadyAudio(at: handoffNow), + "collected bytes are not ready until the empty final") +check(handoff.takeReadyAudio(now: handoffNow).isEmpty && handoff.byteCount == 4 && handoff.phase == .collecting, + "taking before completion cannot drain collecting audio") +handoff.complete(empty: true, now: handoffNow) +check(handoff.phase == .ready && handoff.hasReadyAudio(at: handoffNow) && handoff.byteCount == 4, + "empty final makes collected audio available") +handoff.append(secondPCM) +check(handoff.byteCount == 4, "audio after completion cannot enter the handoff") +check(handoff.takeReadyAudio(now: handoffNow) == firstPCM, "empty final hands off the original PCM bytes") +check(handoff.phase == .idle && handoff.byteCount == 0 && !handoff.hasReadyAudio(at: handoffNow), + "taking ready audio clears state and storage") +check(handoff.takeReadyAudio(now: handoffNow).isEmpty, "an empty final can only be drained once") +handoff.append(secondPCM) +check(handoff.byteCount == 0, "audio after draining cannot start another handoff") + +handoff.begin() +handoff.append(Data([0, 1, 2])) +handoff.append(Data([3, 4, 5, 6, 7, 8, 9])) +handoff.append(Data([10, 11])) +check(handoff.byteCount == 8, "collecting never exceeds the byte capacity") +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == Data([0, 1, 2, 3, 4, 5, 6, 7]), + "overflow retains the stream prefix across packet boundaries") + +var oddCapacity = OpenGlassSpeechHandoffBuffer(capacity: 5) +oddCapacity.begin() +oddCapacity.append(Data([0, 1, 2])) +oddCapacity.append(Data([3, 4, 5])) +check(oddCapacity.byteCount == 4, "odd capacity rounds down to a complete PCM16 sample") +oddCapacity.complete(empty: true, now: handoffNow) +check(oddCapacity.takeReadyAudio(now: handoffNow) == firstPCM, + "odd packet boundaries preserve prefix order and sample alignment") +var tinyCapacity = OpenGlassSpeechHandoffBuffer(capacity: 1) +tinyCapacity.begin() +tinyCapacity.append(firstPCM) +tinyCapacity.complete(empty: true, now: handoffNow) +check(tinyCapacity.phase == .ready && tinyCapacity.byteCount == 0 && !tinyCapacity.hasReadyAudio(at: handoffNow), + "sub-sample capacity can complete without retaining a partial sample") +check(tinyCapacity.takeReadyAudio(now: handoffNow).isEmpty && tinyCapacity.phase == .idle, + "an empty ready buffer can be drained and cleared") +var negativeCapacity = OpenGlassSpeechHandoffBuffer(capacity: -3) +negativeCapacity.begin() +negativeCapacity.append(firstPCM) +check(negativeCapacity.byteCount == 0, "negative capacity is safely treated as zero") +var defaultCapacity = OpenGlassSpeechHandoffBuffer() +defaultCapacity.begin() +defaultCapacity.append(Data(repeating: 9, count: 70_403)) +check(defaultCapacity.byteCount == 70_400, "default capacity holds at most 2.2 seconds of PCM16") +defaultCapacity.clear() + +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: false, now: handoffNow) +check(handoff.phase == .idle && handoff.byteCount == 0 && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "a nonempty final discards buffered audio before inference or TTS") +handoff.append(secondPCM) +check(handoff.byteCount == 0, "nonempty completion rejects late audio") +handoff.begin() +handoff.append(firstPCM) +handoff.clear() +handoff.complete(empty: true, now: handoffNow) +check(handoff.phase == .idle && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "stop or error before finalization prevents a late empty final from reviving audio") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.clear() +check(handoff.phase == .idle && handoff.byteCount == 0 && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "stop or error also discards audio already ready for handoff") + +handoff.begin() +handoff.append(firstPCM) +handoff.begin() +check(handoff.phase == .collecting && handoff.byteCount == 0, + "starting a new collection discards an unfinished old collection") +handoff.append(secondPCM) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == secondPCM, "a new round never replays the previous round's PCM") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.begin() +handoff.append(secondPCM) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == secondPCM, "begin also isolates an old ready handoff from a new round") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.complete(empty: false, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow).isEmpty && handoff.phase == .idle, + "a nonempty outcome invalidates a previously ready handoff") + +handoff.begin() +handoff.append(Data([0x34])) +handoff.append(Data([0x12])) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == Data([0x34, 0x12]), + "two half-sample packets concatenate without losing either byte") +handoff.begin() +handoff.append(Data([0x34])) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == Data([0x34]), + "an incomplete final sample is preserved for the next live packet") + +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +check(handoff.hasReadyAudio(at: handoffNow + 0.499), "handoff remains ready within its short transfer window") +check(handoff.hasReadyAudio(at: handoffNow + 0.5), "handoff is valid at the half-second deadline") +check(!handoff.hasReadyAudio(at: handoffNow + 0.501), "handoff expires after the half-second deadline") +check(handoff.takeReadyAudio(now: handoffNow + 0.501).isEmpty && handoff.phase == .idle && handoff.byteCount == 0, + "delayed frame or admission gate cannot replay expired PCM") +handoff.begin() +handoff.append(secondPCM) +handoff.complete(empty: true, now: handoffNow + 1) +check(!handoff.hasReadyAudio(at: handoffNow), "a timestamp before completion cannot make a handoff ready") +check(handoff.takeReadyAudio(now: handoffNow + 1.5) == secondPCM, + "new completion owns a fresh deadline and can drain exactly at it") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.clear() +check(!handoff.hasReadyAudio(at: handoffNow + 0.1) && handoff.takeReadyAudio(now: handoffNow + 0.1).isEmpty, + "clear invalidates an unexpired ready handoff") +print("Passed \(checks - checksBeforeHandoff) OpenGlass speech-handoff checks") +print("Passed \(checks) OpenGlass voice-loop, diagnostics and handoff checks") diff --git a/ios/upstream-lock.json b/ios/upstream-lock.json new file mode 100644 index 0000000..9c57071 --- /dev/null +++ b/ios/upstream-lock.json @@ -0,0 +1,19 @@ +{ + "repository": "https://github.com/OpenBMB/MiniCPM-V-Apps.git", + "commit": "fa9d837e3d38dfde7a9484ec1cc6bc2e5bc9b420", + "submodule": { + "path": "llama.cpp-omni", + "repository": "https://github.com/tc-mb/llama.cpp-omni.git", + "commit": "96bbf61710d44421ea54e63f5194f9b17c20d877" + }, + "xcode": "26.1.1", + "bundle_identifier": "org.opensqz.openglass.ios", + "display_name": "OpenGlass", + "version": "1.0", + "swift_packages": { + "hxphotopicker": "69510f1e0c9bd1470ee67dd4c765c1f94d872098", + "kingfisher": "2ef543ee21d63734e1c004ad6c870255e8716c50", + "snapkit": "2842e6e84e82eb9a8dac0100ca90d9444b0307f4", + "ziparchive": "df35718ea19a94e015b91dc4881dee028ce4cdba" + } +} diff --git a/ios/windows/.gitignore b/ios/windows/.gitignore new file mode 100644 index 0000000..a230a78 --- /dev/null +++ b/ios/windows/.gitignore @@ -0,0 +1,2 @@ +.venv/ +__pycache__/ diff --git a/ios/windows/common.ps1 b/ios/windows/common.ps1 new file mode 100644 index 0000000..b0c20c9 --- /dev/null +++ b/ios/windows/common.ps1 @@ -0,0 +1,84 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Resolve-OpenGlassAdb { + param([string]$Adb) + if (-not $Adb) { + $found = Get-Command adb.exe -ErrorAction SilentlyContinue + if ($found) { $Adb = $found.Source } + } + if (-not $Adb) { + $candidate = Join-Path $env:USERPROFILE 'Downloads\platform-tools\adb.exe' + if (Test-Path -LiteralPath $candidate -PathType Leaf) { $Adb = $candidate } + } + if (-not $Adb -or -not (Test-Path -LiteralPath $Adb -PathType Leaf)) { + throw 'ADB not found. Install Android Platform Tools and pass -Adb .' + } + return (Resolve-Path -LiteralPath $Adb).Path +} + +function Invoke-OpenGlassAdb { + param([string]$Adb, [string[]]$Arguments) + $output = @(& $Adb @Arguments 2>&1) + if ($LASTEXITCODE -ne 0) { throw "ADB failed ($LASTEXITCODE): $($output -join [Environment]::NewLine)" } + return ($output -join [Environment]::NewLine) +} + +function Get-OpenGlassRokid { + param([string]$Adb, [string]$Serial, [string]$Package, [switch]$UsbOnly) + $rows = Invoke-OpenGlassAdb $Adb @('devices', '-l') + $ready = @($rows -split '\r?\n' | ForEach-Object { + if ($_ -match '^([A-Za-z0-9_.:-]+)\s+device(?:\s|$)') { $Matches[1] } + }) + if ($UsbOnly) { $ready = @($ready | Where-Object { $_ -notmatch ':' }) } + if (-not $Serial) { + if ($ready.Count -ne 1) { throw 'Expected exactly one authorized Android device. Unlock/authorize it or pass -Serial from adb devices -l.' } + $Serial = $ready[0] + } + if ($Serial -notmatch '^[A-Za-z0-9_.:-]+$' -or $Serial -notin $ready) { + throw 'The selected Rokid device is not authorized/online on the requested transport.' + } + $listing = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'pm', 'list', 'packages') + $installed = @($listing -split '\r?\n' | ForEach-Object { + if ($_ -match '^package:(org\.opensqz\.openglass\.rokid(?:\.(?:debug|wifi))?)\s*$') { $Matches[1] } + }) + if (-not $Package) { + if ($installed.Count -ne 1) { throw 'Expected exactly one OpenGlass Rokid package. Install the collector or pass -Package for the installed debug/release variant.' } + $Package = $installed[0] + } + if ($Package -notin $installed) { throw 'The requested OpenGlass Rokid package is not installed.' } + $info = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'dumpsys', 'package', $Package) + if ($info -notmatch 'versionName=((?:0\.1\.3|0\.2\.0)(?:-(?:debug|wifi))?)(?:\s|$)') { + throw 'Collector version 0.1.3 or 0.2.0 is required. Upgrade or review newer protocol compatibility first.' + } + $version = $Matches[1] + $resolved = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'cmd', 'package', 'resolve-activity', '--brief', '-a', 'android.intent.action.MAIN', '-c', 'android.intent.category.LAUNCHER', $Package) + $activities = @($resolved -split '\r?\n' | Where-Object { $_ -match ('^' + [regex]::Escape($Package) + '/[A-Za-z0-9_.$]+$') }) + if ($activities.Count -ne 1) { throw 'Could not resolve the installed collector launcher activity. No app was started.' } + [pscustomobject]@{ Serial = $Serial; Package = $Package; Version = $version; Activity = $activities[0] } +} + +function Resolve-OpenGlassPython { + param([string]$Python = 'python') + $candidates = @() + if ($Python -eq 'python') { + $repoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) + $candidates += Join-Path $PSScriptRoot '.venv\Scripts\python.exe' + $candidates += Join-Path $repoRoot 'tmp\ios-tools\Scripts\python.exe' + $pathPython = Get-Command python.exe -ErrorAction SilentlyContinue + if ($pathPython) { $candidates += $pathPython.Source } + $candidates += Join-Path $env:USERPROFILE 'miniconda3\python.exe' + $candidates += Join-Path $env:USERPROFILE 'anaconda3\python.exe' + } else { + $explicit = Get-Command $Python -ErrorAction SilentlyContinue + if ($explicit) { $candidates += $explicit.Source } + } + foreach ($candidate in ($candidates | Select-Object -Unique)) { + # Do not launch Windows' Store app-execution alias during a diagnostic. + if ($candidate -match '[\\/]Microsoft[\\/]WindowsApps[\\/]' -or + -not (Test-Path -LiteralPath $candidate -PathType Leaf)) { continue } + $version = @(& $candidate -c 'import sys; assert sys.version_info >= (3, 11); print(sys.executable)' 2>&1) + if ($LASTEXITCODE -eq 0) { return (Resolve-Path -LiteralPath $candidate).Path } + } + throw 'No working Python 3.11+ found (WindowsApps Store aliases are skipped). Pass -Python C:\path\to\python.exe, then install ios/windows/requirements.txt into that environment.' +} diff --git a/ios/windows/configure-rokid.ps1 b/ios/windows/configure-rokid.ps1 new file mode 100644 index 0000000..888d4d1 --- /dev/null +++ b/ios/windows/configure-rokid.ps1 @@ -0,0 +1,45 @@ +<# Start the installed Rokid collector against the iPhone on the same LAN. #> +[CmdletBinding(DefaultParameterSetName = 'Manual')] +param( + [Parameter(Mandatory = $true, ParameterSetName = 'Manual')][string]$IPhoneAddress, + [Parameter(Mandatory = $true, ParameterSetName = 'Automatic')][switch]$AutoDiscover, + [ValidateRange(1, 65535)][int]$Port = 18080, + [string]$Adb, + [string]$Serial, + [string]$Package, + [switch]$SkipReceiverCheck +) +. (Join-Path $PSScriptRoot 'common.ps1') +$adbPath = Resolve-OpenGlassAdb $Adb +$device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package +if ($AutoDiscover) { + if ($device.Version -notmatch '^(\d+)\.(\d+)\.(\d+)' -or + [version]$Matches[0] -lt [version]'0.2.0') { + throw 'Automatic Wi-Fi discovery requires OpenGlassRokidSensor 0.2.0 or newer. Install the new APK first.' + } + $result = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'shell', 'am', 'start', '-S', '-W', '-n', $device.Activity, '--ez', 'auto_discover', 'true') + if ($result -match 'Error:|Exception') { throw "Collector launch failed: $result" } + Write-Host 'Automatic Wi-Fi discovery enabled. The glasses verify /health before sending images or audio.' + Write-Host 'Join the iPhone hotspot or the same Wi-Fi in the glasses settings, and keep OpenGlass receiving on iPhone.' + Write-Host 'No Windows relay is used in this mode. Verify both streams before unplugging USB.' + return +} +$address = $null +if (-not [System.Net.IPAddress]::TryParse($IPhoneAddress, [ref]$address) -or + $address.AddressFamily -ne [System.Net.Sockets.AddressFamily]::InterNetwork -or + [System.Net.IPAddress]::IsLoopback($address) -or $address.ToString() -eq '0.0.0.0') { + throw 'Pass the iPhone Wi-Fi IPv4 address, not localhost, a URL, or the glasses address.' +} +$baseUrl = 'http://{0}:{1}' -f $address.ToString(), $Port +if (-not $SkipReceiverCheck) { + try { $health = Invoke-RestMethod -Uri "$baseUrl/health" -TimeoutSec 5 } + catch { throw "iPhone /health is unreachable at $baseUrl. Open OpenGlass on iPhone and start its receiver on the same LAN. $($_.Exception.Message)" } + if ($health.PSObject.Properties.Name -notcontains 'service' -or $health.service -ne 'openglass-ios') { + throw 'The endpoint is not the OpenGlass iOS receiver. Nothing was changed.' + } +} +$result = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'shell', 'am', 'start', '-S', '-W', '-n', $device.Activity, '--es', 'pc_base_url', $baseUrl) +if ($result -match 'Error:|Exception') { throw "Collector launch failed: $result" } +Write-Host "Started $($device.Package) $($device.Version) -> $baseUrl" +Write-Host 'Check that image_count and audio_packets_in increase on iPhone. The URL is saved by the collector.' +Write-Host 'Wi-Fi was not enabled or reconfigured by this script. Both devices must already have network access.' diff --git a/ios/windows/diagnose.ps1 b/ios/windows/diagnose.ps1 new file mode 100644 index 0000000..b061b62 --- /dev/null +++ b/ios/windows/diagnose.ps1 @@ -0,0 +1,62 @@ +<# Read-only installation diagnostics. Does not sign, install, pair, or start apps. #> +param([string]$Adb, [string]$Serial, [string]$Package, [string]$Python = 'python', [string]$Ipa) +. (Join-Path $PSScriptRoot 'common.ps1') +Write-Host 'OpenGlass Windows / iPhone preflight' +Write-Host 'Apple credentials are not requested or read by this script.' + +try { + $adbPath = Resolve-OpenGlassAdb $Adb + $device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package + Write-Host "[OK] Rokid collector: $($device.Package), version $($device.Version)" + Write-Host " Activity: $($device.Activity)" + $wifi = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'shell', 'settings', 'get', 'global', 'wifi_on') + Write-Host " Wi-Fi setting: $wifi (0=off; USB relay does not require Wi-Fi)" + $reverse = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'reverse', '--list') + if ($reverse) { Write-Host " Existing reverse mappings (left untouched):`n$reverse" } +} catch { Write-Warning "Rokid: $($_.Exception.Message)" } + +if (Get-Command Get-PnpDevice -ErrorAction SilentlyContinue) { + $phones = @(Get-PnpDevice -PresentOnly -ErrorAction SilentlyContinue | Where-Object { $_.FriendlyName -match 'iPhone|Apple Mobile Device' }) + Write-Host "[INFO] Windows PnP iPhone/Apple Mobile Device entries: $($phones.Count)" + foreach ($phone in $phones) { Write-Host " $($phone.FriendlyName): $($phone.Status)" } + Write-Host ' PnP detection alone does not prove trust, app installation, or a running listener.' +} +$appleService = @(Get-Service -ErrorAction SilentlyContinue | Where-Object { $_.Name -eq 'Apple Mobile Device Service' -or $_.DisplayName -eq 'Apple Mobile Device Service' }) +if ($appleService.Count) { Write-Host "[INFO] Apple Mobile Device Service: $($appleService[0].Status)" } +else { Write-Warning 'Apple Mobile Device Service was not found. Follow the official AltServer Windows prerequisites.' } +foreach ($processName in @('AltServer', 'iTunes', 'iCloud')) { + $running = @(Get-Process -Name $processName -ErrorAction SilentlyContinue).Count -gt 0 + Write-Host "[INFO] $processName running: $running" +} +try { + $pythonPath = Resolve-OpenGlassPython $Python + $moduleVersion = @(& $pythonPath -c 'from importlib.metadata import version; print(version("pymobiledevice3"))' 2>&1) + if ($LASTEXITCODE -ne 0) { throw 'pymobiledevice3 is not installed in this Python environment.' } + Write-Host "[INFO] pymobiledevice3: $($moduleVersion -join ' ')" + Write-Host '[INFO] USB iPhone identifiers (local output; omit these when sharing logs):' + & $pythonPath -m pymobiledevice3 usbmux list --usb --simple + if ($LASTEXITCODE -ne 0) { Write-Warning 'usbmux discovery failed. Check USB, unlock/trust, and Apple Mobile Device Service.' } +} catch { Write-Warning "USB tools: $($_.Exception.Message)" } + +if (Get-Command Get-NetTCPConnection -ErrorAction SilentlyContinue) { + $listeners = @(Get-NetTCPConnection -State Listen -LocalPort 18080 -ErrorAction SilentlyContinue) + if ($listeners.Count) { + Write-Warning 'Windows port 18080 is occupied. Stop its owning session yourself, or use -LocalPort 18081 for the USB relay.' + $listeners | Select-Object LocalAddress, LocalPort, OwningProcess | Format-Table + } else { Write-Host '[OK] No listener currently found on Windows port 18080.' } +} +if ($Ipa) { + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead((Resolve-Path -LiteralPath $Ipa).Path) + try { + $names = @($archive.Entries | ForEach-Object { $_.FullName }) + $appPlists = @($names | Where-Object { $_ -match '^Payload/[^/]+\.app/Info\.plist$' }) + if ($appPlists.Count -ne 1) { throw 'IPA must contain one Payload/.app/Info.plist.' } + $provision = @($names | Where-Object { $_ -match '^Payload/[^/]+\.app/embedded\.mobileprovision$' }).Count -gt 0 + $signature = @($names | Where-Object { $_ -match '^Payload/[^/]+\.app/_CodeSignature/CodeResources$' }).Count -gt 0 + Write-Host "[INFO] IPA layout valid; embedded profile=$provision, signature resources=$signature" + Write-Host ' File presence is not signature verification. CI unsigned IPA must be re-signed using AltStore/AltServer before installation.' + } finally { $archive.Dispose() } +} +Write-Host 'On iPhone: unlock, Trust This Computer, enable Developer Mode, then trust your signing profile when prompted.' +Write-Host 'For a previously unopenable app, check the actual error/expiry first; this script does not uninstall it.' diff --git a/ios/windows/download-artifact.ps1 b/ios/windows/download-artifact.ps1 new file mode 100644 index 0000000..ebd8c79 --- /dev/null +++ b/ios/windows/download-artifact.ps1 @@ -0,0 +1,61 @@ +<# Download one successful, explicitly selected GitHub Actions run. Never uploads or starts CI. #> +param( + [Parameter(Mandatory = $true)][ValidatePattern('^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$')][string]$Repository, + [Parameter(Mandatory = $true)][ValidateRange(1, [long]::MaxValue)][long]$RunId, + [string]$ArtifactName, + [string]$Destination +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$gh = (Get-Command gh -ErrorAction Stop).Source +$runJson = @(& $gh api "repos/$Repository/actions/runs/$RunId" 2>&1) +if ($LASTEXITCODE -ne 0) { throw 'Cannot read the run. Check the repository/run ID and use gh auth login yourself if needed.' } +$run = ($runJson -join [Environment]::NewLine) | ConvertFrom-Json +if ($run.status -ne 'completed' -or $run.conclusion -ne 'success') { throw 'The selected workflow run has not completed successfully.' } +if ($run.path -notmatch '(^|/)ios-ipa\.yml(?:@.*)?$') { throw 'The run is not the ios-ipa.yml workflow. Select the iOS build run.' } +$artifactJson = @(& $gh api "repos/$Repository/actions/runs/$RunId/artifacts?per_page=100" 2>&1) +if ($LASTEXITCODE -ne 0) { throw 'Cannot list the run artifacts.' } +$artifactResponse = ($artifactJson -join [Environment]::NewLine) | ConvertFrom-Json +$candidates = @($artifactResponse.artifacts | Where-Object { + -not $_.expired -and $_.name -match '^OpenGlass-iOS-unsigned-[0-9]+$' +}) +if ($ArtifactName) { $candidates = @($candidates | Where-Object { $_.name -eq $ArtifactName }) } +if ($candidates.Count -ne 1) { throw 'Expected exactly one non-expired OpenGlass iOS artifact. Pass -ArtifactName or inspect the selected run.' } +if (-not $Destination) { + $Destination = Join-Path (Split-Path -Parent $PSScriptRoot) "artifacts\run-$RunId" +} +$fullDestination = [System.IO.Path]::GetFullPath($Destination) +if (Test-Path -LiteralPath $fullDestination) { throw "Destination already exists; choose a new empty output path: $fullDestination" } +New-Item -ItemType Directory -Path $fullDestination | Out-Null +& $gh run download "$RunId" --repo $Repository --name $candidates[0].name --dir $fullDestination +if ($LASTEXITCODE -ne 0) { throw "Artifact download failed. Partial output remains for inspection: $fullDestination" } +$ipa = Join-Path $fullDestination 'OpenGlass-1.0.ipa' +if (-not (Test-Path -LiteralPath $ipa -PathType Leaf)) { throw "The artifact is missing OpenGlass-1.0.ipa: $fullDestination" } +$manifestPath = Join-Path $fullDestination 'build-info.json' +if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { throw 'The artifact is missing build-info.json.' } +$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json +foreach ($field in @('commit', 'signed', 'bundle_identifier')) { + if ($manifest.PSObject.Properties.Name -notcontains $field) { throw "build-info.json is missing $field." } +} +if ($manifest.commit -cne $run.head_sha) { throw 'Artifact manifest commit does not match the selected Actions run.' } +if ($manifest.signed -isnot [bool] -or $manifest.signed) { throw 'Expected the manifest to declare signed=false (JSON boolean).' } +if ($manifest.bundle_identifier -cne 'org.opensqz.openglass.ios') { throw 'Artifact manifest has an unexpected iOS bundle identifier.' } +Add-Type -AssemblyName System.IO.Compression.FileSystem +$archive = [System.IO.Compression.ZipFile]::OpenRead($ipa) +try { + $infoEntries = @($archive.Entries | Where-Object { $_.FullName -ceq 'Payload/OpenGlass.app/Info.plist' }) + if ($infoEntries.Count -ne 1 -or $infoEntries[0].Length -le 0 -or $infoEntries[0].Length -gt 1MB) { + throw 'IPA must contain one non-empty Payload/OpenGlass.app/Info.plist of reasonable size.' + } + # Read the compressed entry without extracting any archive path to disk. + $infoStream = $infoEntries[0].Open() + try { $infoStream.CopyTo([System.IO.Stream]::Null) } + finally { $infoStream.Dispose() } +} finally { $archive.Dispose() } +Write-Host "Downloaded unsigned IPA: $ipa" +Write-Host 'Verified manifest commit, unsigned status, bundle identifier, and readable IPA Info.plist entry.' +Write-Host "Run: $($run.html_url)" +Write-Host "Commit: $($run.head_sha)" +Get-FileHash -LiteralPath $ipa -Algorithm SHA256 | Format-List +Write-Host 'Next: hold Shift while clicking the Windows AltServer tray icon, choose Sideload .ipa, and sign/install with your own Apple account in its UI.' +Write-Host 'AltStore on iPhone is optional and uses a free-account app slot. Direct AltServer installs require manual re-signing every 7 days.' diff --git a/ios/windows/requirements.txt b/ios/windows/requirements.txt new file mode 100644 index 0000000..522ea59 --- /dev/null +++ b/ios/windows/requirements.txt @@ -0,0 +1,2 @@ +# API verified against the upstream v11.32.0 tag; use a dedicated Python 3.11+ venv. +pymobiledevice3==11.32.0 diff --git a/ios/windows/start-usb-relay.ps1 b/ios/windows/start-usb-relay.ps1 new file mode 100644 index 0000000..2a68d50 --- /dev/null +++ b/ios/windows/start-usb-relay.ps1 @@ -0,0 +1,23 @@ +<# Keep this terminal open. Ctrl+C removes only the relay mapping created by this run. #> +param( + [string]$Adb, + [string]$Serial, + [string]$Package, + [string]$IPhoneUdid, + [string]$Python = 'python', + [ValidateRange(1, 65535)][int]$LocalPort = 18080, + [ValidateRange(1, 65535)][int]$IPhonePort = 18080, + [ValidateRange(1, 65535)][int]$RokidPort = 18080 +) +. (Join-Path $PSScriptRoot 'common.ps1') +$adbPath = Resolve-OpenGlassAdb $Adb +$pythonPath = Resolve-OpenGlassPython $Python +$device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package -UsbOnly +$relayArgs = @( + (Join-Path $PSScriptRoot 'usb_relay.py'), + '--adb', $adbPath, '--rokid-serial', $device.Serial, '--activity', $device.Activity, + '--local-port', "$LocalPort", '--iphone-port', "$IPhonePort", '--rokid-port', "$RokidPort" +) +if ($IPhoneUdid) { $relayArgs += @('--iphone-udid', $IPhoneUdid) } +& $pythonPath @relayArgs +if ($LASTEXITCODE -ne 0) { throw "USB relay exited with code $LASTEXITCODE. Read the diagnostic above; no unrelated processes were stopped." } diff --git a/ios/windows/test_usb_relay.py b/ios/windows/test_usb_relay.py new file mode 100644 index 0000000..54ecb46 --- /dev/null +++ b/ios/windows/test_usb_relay.py @@ -0,0 +1,134 @@ +"""Safety/lifecycle checks with fake devices; never invokes ADB or usbmuxd.""" + +import argparse +import asyncio +import signal +import socket +import types +import unittest +from unittest.mock import patch + +import usb_relay + + +class FakeForwarder: + instances = [] + + def __init__(self, serial, dst_port, src_port, listening_event, usbmux_connection_type): + assert usbmux_connection_type == "USB" + self.src_port = src_port + self.ready = listening_event + self.stopped = asyncio.Event() + self.server = None + self.instances.append(self) + + async def start(self, address): + self.server = await asyncio.start_server(lambda r, w: w.close(), address, self.src_port) + self.ready.set() + try: + await self.stopped.wait() + finally: + self.server.close() + await self.server.wait_closed() + + def stop(self): + self.stopped.set() + + +class RelayTests(unittest.TestCase): + def setUp(self): + FakeForwarder.instances = [] + self.args = argparse.Namespace( + adb="not-called", rokid_serial="TESTDEVICE", activity="org.opensqz.openglass.rokid.debug/.MainActivity", + iphone_udid=None, local_port=0, iphone_port=18080, rokid_port=18080, + ) + self.calls = [] + self.maps = [] + self.change_mapping = False + self.reject_mapping = False + + async def list_devices(self): + return [types.SimpleNamespace(serial="TESTPHONE", is_usb=True)] + + def fake_adb(self, args, *command): + self.calls.append(command) + if command == ("reverse", "--list"): + return "\n".join(f"UsbFfs {remote} {local}" for remote, local in self.maps) + if command[:2] == ("reverse", "--no-rebind"): + if self.reject_mapping: + raise RuntimeError("mapping was claimed by another session") + self.maps = [(command[2], command[3])] + if command[:3] == ("shell", "am", "start"): + if self.change_mapping: + self.maps = [("tcp:18080", "tcp:19000")] + asyncio.get_running_loop().call_soon(signal.getsignal(signal.SIGINT), signal.SIGINT, None) + if command[:2] == ("reverse", "--remove"): + self.maps = [] + return "" + + def execute(self, health=None): + package = types.ModuleType("pymobiledevice3") + package.usbmux = types.SimpleNamespace(list_devices=self.list_devices) + forwarder = types.ModuleType("pymobiledevice3.tcp_forwarder") + forwarder.UsbmuxTcpForwarder = FakeForwarder + modules = {"pymobiledevice3": package, "pymobiledevice3.tcp_forwarder": forwarder} + with patch.dict("sys.modules", modules), patch.object(usb_relay, "adb", self.fake_adb), patch.object( + usb_relay, "read_health", side_effect=health or (lambda port: {"service": "openglass-ios"}) + ): + asyncio.run(usb_relay.run(self.args)) + + def test_preexisting_reverse_aborts_without_changes(self): + self.maps = [("tcp:18080", "tcp:12345")] + with self.assertRaisesRegex(RuntimeError, "already has"): + self.execute() + self.assertEqual(self.calls, [("reverse", "--list")]) + self.assertEqual(FakeForwarder.instances, []) + + def test_busy_port_does_not_touch_mapping(self): + with socket.socket() as occupied: + occupied.bind(("127.0.0.1", 0)) + occupied.listen() + self.args.local_port = occupied.getsockname()[1] + with self.assertRaises(OSError): + self.execute() + self.assertEqual(self.calls, [("reverse", "--list")]) + + def test_unreachable_receiver_closes_listener_without_launch(self): + with self.assertRaisesRegex(RuntimeError, "keep the app|Keep the app"): + self.execute(health=ConnectionRefusedError("app not running")) + self.assertEqual(self.calls, [("reverse", "--list")]) + self.assertFalse(FakeForwarder.instances[0].server.is_serving()) + + def test_success_launch_and_cleanup_only_own_mapping(self): + self.execute() + self.assertIn(("reverse", "--no-rebind", "tcp:18080", "tcp:0"), self.calls) + self.assertIn(("reverse", "--remove", "tcp:18080"), self.calls) + self.assertTrue(any(command[:3] == ("shell", "am", "start") for command in self.calls)) + self.assertEqual(self.maps, []) + self.assertFalse(FakeForwarder.instances[0].server.is_serving()) + + def test_replaced_mapping_is_left_untouched(self): + self.change_mapping = True + self.execute() + self.assertNotIn(("reverse", "--remove", "tcp:18080"), self.calls) + self.assertEqual(self.maps, [("tcp:18080", "tcp:19000")]) + + def test_no_rebind_race_failure_does_not_remove_mapping(self): + self.reject_mapping = True + with self.assertRaisesRegex(RuntimeError, "another session"): + self.execute() + self.assertNotIn(("reverse", "--remove", "tcp:18080"), self.calls) + self.assertFalse(FakeForwarder.instances[0].server.is_serving()) + + def test_non_iphone_health_is_rejected(self): + response = types.SimpleNamespace(read=lambda limit: b'{"service":"other"}') + context = unittest.mock.MagicMock() + context.__enter__.return_value = response + opener = types.SimpleNamespace(open=lambda *a, **kw: context) + with patch.object(usb_relay.urllib.request, "build_opener", return_value=opener): + with self.assertRaisesRegex(RuntimeError, "not the OpenGlass iOS"): + usb_relay.read_health(18080) + + +if __name__ == "__main__": + unittest.main() diff --git a/ios/windows/usb_relay.py b/ios/windows/usb_relay.py new file mode 100644 index 0000000..840dc24 --- /dev/null +++ b/ios/windows/usb_relay.py @@ -0,0 +1,175 @@ +"""Relay a Rokid USB collector to an iPhone TCP listener without Wi-Fi. + +The model runs on iPhone. Windows forwards bytes only. Uses upstream +pymobiledevice3 v11.32.0 UsbmuxTcpForwarder and a scoped ADB reverse mapping. +No signing, Apple account access, automatic pairing, or process-wide cleanup. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import logging +import re +import signal +import subprocess +import sys +import urllib.request + + +def adb(args: argparse.Namespace, *command: str) -> str: + result = subprocess.run( + [args.adb, "-s", args.rokid_serial, *command], + capture_output=True, text=True, timeout=20, check=False, + ) + if result.returncode: + raise RuntimeError(f"ADB failed: {result.stderr.strip() or result.stdout.strip()}") + return result.stdout.strip() + + +def reverse_pairs(text: str) -> list[tuple[str, str]]: + """ADB prints one line per mapping: transport remote local.""" + pairs = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) >= 2: + pairs.append((fields[-2], fields[-1])) + return pairs + + +def read_health(port: int) -> dict: + # Ignore system HTTP proxies for a loopback-only request. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(f"http://127.0.0.1:{port}/health", timeout=5) as response: + payload = json.loads(response.read(64 * 1024)) + if not isinstance(payload, dict) or payload.get("service") != "openglass-ios": + raise RuntimeError("/health is not the OpenGlass iOS receiver.") + return payload + + +async def run(args: argparse.Namespace) -> None: + try: + from pymobiledevice3 import usbmux + from pymobiledevice3.tcp_forwarder import UsbmuxTcpForwarder + except ImportError as exc: + raise RuntimeError("Install ios/windows/requirements.txt into this Python environment first.") from exc + + devices = [device for device in await asyncio.wait_for(usbmux.list_devices(), 10) if device.is_usb] + if args.iphone_udid: + devices = [device for device in devices if device.matches_udid(args.iphone_udid)] + if len(devices) != 1: + raise RuntimeError("Expected one USB iPhone. Unlock/trust it, check Apple Mobile Device Service, or pass --iphone-udid.") + remote, local = f"tcp:{args.rokid_port}", f"tcp:{args.local_port}" + if any(pair[0] == remote for pair in reverse_pairs(adb(args, "reverse", "--list"))): + raise RuntimeError(f"Rokid {remote} already has an ADB reverse mapping. Stop its owning session or choose --rokid-port. No mapping was replaced.") + + ready = asyncio.Event() + stop = asyncio.Event() + forwarder = UsbmuxTcpForwarder( + devices[0].serial, args.iphone_port, args.local_port, + listening_event=ready, usbmux_connection_type="USB", + ) + task = asyncio.create_task(forwarder.start(address="127.0.0.1")) + readiness = asyncio.create_task(ready.wait()) + mapping_created = False + previous_handlers = {} + loop = asyncio.get_running_loop() + # signal.signal works with Windows' default Proactor loop; add_signal_handler does not. + for sig in (signal.SIGINT, signal.SIGTERM): + previous_handlers[sig] = signal.getsignal(sig) + signal.signal(sig, lambda *_: loop.call_soon_threadsafe(stop.set)) + try: + done, _ = await asyncio.wait({task, readiness}, timeout=10, return_when=asyncio.FIRST_COMPLETED) + if task in done: + await task # Surface bind/port-conflict failures without touching ADB state. + raise RuntimeError("USB listener stopped before startup.") + if readiness not in done: + raise RuntimeError("Timed out starting the local USB listener.") + if stop.is_set(): + return + try: + health = await asyncio.to_thread(read_health, args.local_port) + except Exception as exc: + raise RuntimeError("iPhone OpenGlass /health is unavailable. Keep the app in front and start its receiver. USB discovery alone does not mean the app is installed/running.") from exc + if stop.is_set(): + return + # --no-rebind closes the race between the preflight list and mapping creation. + adb(args, "reverse", "--no-rebind", remote, local) + mapping_created = True + result = adb(args, "shell", "am", "start", "-S", "-W", "-n", args.activity, + "--es", "pc_base_url", f"http://127.0.0.1:{args.rokid_port}") + if "Error:" in result or "Exception" in result: + raise RuntimeError(f"Collector launch failed: {result}") + print(f"Rokid:{args.rokid_port} -> Windows loopback:{args.local_port} -> iPhone USB:{args.iphone_port}", flush=True) + print(f"Receiver ready: image_count={health.get('image_count', 0)}, audio_packets_in={health.get('audio_packets_in', 0)}", flush=True) + print("Keep this terminal and the iPhone app open. Ctrl+C stops this relay only.", flush=True) + stopping = asyncio.create_task(stop.wait()) + try: + done, _ = await asyncio.wait({task, stopping}, return_when=asyncio.FIRST_COMPLETED) + if task in done: + await task + finally: + stopping.cancel() + with contextlib.suppress(asyncio.CancelledError): + await stopping + finally: + if mapping_created: + try: + pairs = reverse_pairs(adb(args, "reverse", "--list")) + if (remote, local) in pairs: + adb(args, "reverse", "--remove", remote) + elif any(pair[0] == remote for pair in pairs): + print("Mapping changed by another session; leaving it untouched.", file=sys.stderr) + except Exception as exc: + print(f"Could not remove this session's reverse mapping (device may be unplugged): {exc}", file=sys.stderr) + forwarder.stop() + if not task.done(): + try: + await asyncio.wait_for(asyncio.shield(task), timeout=5) + except asyncio.TimeoutError: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + readiness.cancel() + with contextlib.suppress(asyncio.CancelledError): + await readiness + for sig, previous in previous_handlers.items(): + signal.signal(sig, previous) + + +def port(value: str) -> int: + number = int(value) + if not 1 <= number <= 65535: + raise argparse.ArgumentTypeError("port must be between 1 and 65535") + return number + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adb", required=True) + parser.add_argument("--rokid-serial", required=True) + parser.add_argument("--activity", required=True, help="Resolved installed Android launcher component") + parser.add_argument("--iphone-udid") + parser.add_argument("--local-port", type=port, default=18080) + parser.add_argument("--iphone-port", type=port, default=18080) + parser.add_argument("--rokid-port", type=port, default=18080) + args = parser.parse_args() + if not re.fullmatch(r"[A-Za-z0-9_.-]+", args.rokid_serial): + parser.error("Rokid must use an explicit USB serial, not a wireless ADB address") + if not re.fullmatch(r"org\.opensqz\.openglass\.rokid(?:\.debug)?/[A-Za-z0-9_.$]+", args.activity): + parser.error("activity must be the resolved OpenGlass Rokid launcher") + logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") + try: + asyncio.run(run(args)) + return 0 + except KeyboardInterrupt: + return 0 + except Exception as exc: + print(f"USB relay stopped: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rokid_app/.gitignore b/rokid_app/.gitignore index 86e3d4d..4244a57 100644 --- a/rokid_app/.gitignore +++ b/rokid_app/.gitignore @@ -10,3 +10,4 @@ sessions/ *.keystore *.jks *.before-* +.build-tools/ diff --git a/rokid_app/OpenGlassRokidSensor/app/build.gradle b/rokid_app/OpenGlassRokidSensor/app/build.gradle index b0187ec..2d97f90 100644 --- a/rokid_app/OpenGlassRokidSensor/app/build.gradle +++ b/rokid_app/OpenGlassRokidSensor/app/build.gradle @@ -11,8 +11,9 @@ android { applicationId 'org.opensqz.openglass.rokid' minSdk 28 targetSdk 34 - versionCode 4 - versionName '0.1.3' + versionCode 8 + versionName '0.2.0' + manifestPlaceholders = [appLabel: '@string/app_name'] buildConfigField 'String', 'PC_BASE_URL', '"http://127.0.0.1:18080"' buildConfigField 'String', 'PC_WS_URL', '"ws://127.0.0.1:18080/rokid/audio"' @@ -27,6 +28,14 @@ android { applicationIdSuffix '.debug' versionNameSuffix '-debug' } + wifi { + initWith debug + applicationIdSuffix '.wifi' + versionNameSuffix '-wifi' + manifestPlaceholders = [appLabel: 'OpenGlass WiFi'] + signingConfig signingConfigs.debug + matchingFallbacks = ['debug'] + } release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' @@ -45,4 +54,5 @@ android { dependencies { implementation 'androidx.core:core-ktx:1.13.1' + testImplementation 'junit:junit:4.13.2' } diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml b/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml index 0b38007..1db5f84 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml @@ -5,6 +5,8 @@ + + @@ -14,7 +16,7 @@ android:allowBackup="false" android:hardwareAccelerated="true" android:icon="@android:drawable/sym_def_app_icon" - android:label="@string/app_name" + android:label="${appLabel}" android:networkSecurityConfig="@xml/network_security_config" android:roundIcon="@android:drawable/sym_def_app_icon" android:supportsRtl="true" @@ -23,6 +25,7 @@ diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/ConnectionPolicy.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/ConnectionPolicy.kt new file mode 100644 index 0000000..95a1802 --- /dev/null +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/ConnectionPolicy.kt @@ -0,0 +1,36 @@ +package org.opensqz.openglass.rokid + +import java.net.URI + +internal const val USB_BASE_URL = "http://127.0.0.1:18080" + +/** Keep URL validation independent of Android so the selection rules can be unit tested. */ +internal object ConnectionPolicy { + fun normalizeUrl(value: String): String { + val uri = URI(value.trim().trimEnd('/')) + require(uri.scheme == "http" && !uri.host.isNullOrBlank() && + uri.rawUserInfo == null && uri.rawQuery == null && uri.rawFragment == null && + uri.rawPath.isNullOrEmpty() && (uri.port == -1 || uri.port in 1..65535)) { + "Enter http://:18080 without a path or password" + } + return uri.toASCIIString() + } + + fun isLoopback(url: String): Boolean = + URI(url).host.lowercase() in setOf("127.0.0.1", "localhost", "[::1]", "::1") + + fun websocketUrl(url: String): String = "ws://${URI(url).rawAuthority}/rokid/audio" + + fun receiverMatches(name: String, preferred: String?): Boolean = + name.matches(Regex("OpenGlass-[0-9a-fA-F]{8}")) && + (preferred.isNullOrEmpty() || name == preferred) + + // A previously chosen phone must not silently become a different hotel guest's phone. + fun selectService(available: Set, preferred: String?): String? = when { + !preferred.isNullOrEmpty() -> preferred.takeIf { it in available } + available.size == 1 -> available.first() + else -> null + } + + fun nextDelay(delay: Long): Long = (delay * 2).coerceIn(1_000L, 30_000L) +} diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/EndpointController.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/EndpointController.kt new file mode 100644 index 0000000..3ad8601 --- /dev/null +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/EndpointController.kt @@ -0,0 +1,330 @@ +package org.opensqz.openglass.rokid + +import android.content.Context +import android.net.ConnectivityManager +import android.net.LinkProperties +import android.net.Network +import android.net.NetworkCapabilities +import android.net.NetworkRequest +import android.net.nsd.NsdManager +import android.net.nsd.NsdServiceInfo +import android.net.wifi.WifiManager +import android.os.Handler +import android.os.Looper +import android.util.Log +import org.json.JSONObject +import java.net.HttpURLConnection +import java.net.Inet4Address +import java.net.URI +import java.net.URL +import java.util.concurrent.Executors + +internal data class ServerEndpoint(val baseUrl: String, val network: Network?, val source: String) { + val websocketUrl: String get() = ConnectionPolicy.websocketUrl(baseUrl) + fun connection(path: String): HttpURLConnection = + (network?.openConnection(URL(baseUrl + path)) ?: URL(baseUrl + path).openConnection()) as HttpURLConnection +} + +/** All mutable state belongs to the main looper; only bounded HTTP probes run on the worker. */ +internal class EndpointController( + context: Context, + private val onEndpoint: (ServerEndpoint?) -> Unit, + private val onStatus: (String) -> Unit, +) { + private val app = context.applicationContext + private val prefs = app.getSharedPreferences("pc_connection", Context.MODE_PRIVATE) + private val main = Handler(Looper.getMainLooper()) + private val connectivity = app.getSystemService(ConnectivityManager::class.java) + private val nsd = app.getSystemService(NsdManager::class.java) + private val worker = Executors.newSingleThreadExecutor { r -> Thread(r, "openglass-discovery") } + private var active = false + private var generation = 0 + private var probing = false + private var fixedUrl: String? = null + private var wifi: Network? = null + private var gateway: String? = null + private var endpoint: ServerEndpoint? = null + private var retryDelay = 1_000L + private var discovery: NsdManager.DiscoveryListener? = null + private var multicast: WifiManager.MulticastLock? = null + private val services = linkedMapOf() + private val knownServices = linkedMapOf() + private val resolveQueue = ArrayDeque() + private var resolving = false + private var discoveryStartedAt = 0L + private val probeTask = Runnable { probe() } + + private val networkCallback = object : ConnectivityManager.NetworkCallback() { + override fun onAvailable(network: Network) { + if (!active || network == wifi) return + wifi = network + gateway = null + networkChanged() + } + override fun onLinkPropertiesChanged(network: Network, properties: LinkProperties) { + if (!active || network != wifi) return + val address = properties.routes.firstOrNull { it.isDefaultRoute && it.gateway is Inet4Address } + ?.gateway?.hostAddress + if (address != gateway) { + gateway = address + scheduleProbe(0) + } + } + override fun onLost(network: Network) { + if (!active || network != wifi) return + wifi = null + gateway = null + networkChanged() + } + } + + fun start(explicitUrl: String?) { + if (active) stop() + active = true + fixedUrl = explicitUrl + retryDelay = 1_000L + wifi = connectivity.allNetworks.firstOrNull { + connectivity.getNetworkCapabilities(it)?.hasTransport(NetworkCapabilities.TRANSPORT_WIFI) == true + } + gateway = wifi?.let { connectivity.getLinkProperties(it) }?.routes + ?.firstOrNull { it.isDefaultRoute && it.gateway is Inet4Address }?.gateway?.hostAddress + connectivity.registerNetworkCallback( + NetworkRequest.Builder().addTransportType(NetworkCapabilities.TRANSPORT_WIFI).build(), + networkCallback, main, + ) + if (fixedUrl != null) publish(endpointFor(fixedUrl!!, "manual / USB")) else { + startDiscovery() + scheduleProbe(0) + } + } + + fun stop() { + if (!active) return + active = false + generation++ + main.removeCallbacks(probeTask) + try { connectivity.unregisterNetworkCallback(networkCallback) } catch (_: IllegalArgumentException) { } + stopDiscovery() + publish(null) + } + + fun destroy() { stop(); worker.shutdownNow() } + + fun retry() { + // A WebSocket failure does not make an unverified host eligible for sensor uploads. + if (active && fixedUrl == null) scheduleProbe(1_000) + } + + private fun endpointFor(url: String, source: String) = + ServerEndpoint(url, if (ConnectionPolicy.isLoopback(url)) null else wifi, source) + + private fun networkChanged() { + generation++ + services.clear() + knownServices.clear() + retryDelay = 1_000L + if (fixedUrl != null) { + publish(endpointFor(fixedUrl!!, "manual / USB")) + } else { + publish(null) + stopDiscovery() + startDiscovery() + scheduleProbe(500) + } + } + + private fun publish(value: ServerEndpoint?) { + if (endpoint == value) return + endpoint = value + onEndpoint(value) + } + + private fun scheduleProbe(delay: Long) { + if (!active || fixedUrl != null) return + main.removeCallbacks(probeTask) + main.postDelayed(probeTask, delay) + } + + private fun startDiscovery() { + if (!active || wifi == null || discovery != null) return + discoveryStartedAt = android.os.SystemClock.elapsedRealtime() + val token = generation + val listener = object : NsdManager.DiscoveryListener { + override fun onDiscoveryStarted(type: String) = Unit + override fun onDiscoveryStopped(type: String) = Unit + override fun onStartDiscoveryFailed(type: String, code: Int) { main.post { + if (active && token == generation) { + stopDiscovery() + onStatus("Bonjour unavailable; checking iPhone hotspot / saved address") + } + } } + override fun onStopDiscoveryFailed(type: String, code: Int) = Unit + override fun onServiceFound(service: NsdServiceInfo) { main.post { + if (active && token == generation && + (knownServices.containsKey(service.serviceName) || knownServices.size < 8) && + resolveQueue.none { it.serviceName == service.serviceName } && + service.serviceType.trimEnd('.') == "_openglass._tcp") { + knownServices[service.serviceName] = service + resolveQueue.addLast(service) + resolveNext(token) + } + } } + override fun onServiceLost(service: NsdServiceInfo) { main.post { + if (active && token == generation) { + services.remove(service.serviceName) + knownServices.remove(service.serviceName) + scheduleProbe(500) + } + } } + } + discovery = listener + try { + multicast = app.getSystemService(WifiManager::class.java).createMulticastLock("openglass-discovery") + .apply { setReferenceCounted(false); acquire() } + nsd.discoverServices("_openglass._tcp.", NsdManager.PROTOCOL_DNS_SD, listener) + } catch (e: Exception) { + stopDiscovery() + Log.w("OpenGlassRokid", "Discovery unavailable: ${e.javaClass.simpleName}") + } + } + + private fun stopDiscovery() { + discovery?.let { try { nsd.stopServiceDiscovery(it) } catch (_: Exception) { } } + discovery = null + resolveQueue.clear() + resolving = false + multicast?.let { if (it.isHeld) it.release() } + multicast = null + } + + @Suppress("DEPRECATION") // Glasses run Android versions predating the API 34 service-info callback. + private fun resolveNext(token: Int) { + if (resolving || resolveQueue.isEmpty() || !active || token != generation) return + resolving = true + val item = resolveQueue.removeFirst() + val listener = object : NsdManager.ResolveListener { + override fun onResolveFailed(service: NsdServiceInfo, code: Int) { main.post { + if (token == generation) { resolving = false; resolveNext(token) } + } } + override fun onServiceResolved(service: NsdServiceInfo) { main.post { + if (active && token == generation) { + resolving = false + val host = service.host?.hostAddress + if (host != null && service.port in 1..65535) { + runCatching { URI("http", null, host, service.port, null, null, null).toASCIIString() } + .getOrNull()?.let { url -> + val changed = services[service.serviceName] != url + services[service.serviceName] = url + if (changed) scheduleProbe(500) + } + } + resolveNext(token) + } + } } + } + try { nsd.resolveService(item, listener) } catch (_: Exception) { + resolving = false + resolveNext(token) + } + } + + private fun probe() { + if (!active || fixedUrl != null) return + if (wifi == null) { + publish(null) + onStatus("Join the iPhone hotspot or shared Wi-Fi first") + scheduleProbe(retryDelay) + retryDelay = ConnectionPolicy.nextDelay(retryDelay) + return + } + if (probing) { scheduleProbe(500); return } + if (discovery == null) startDiscovery() + probing = true + val token = generation + val hotspot = gateway?.let { endpointFor("http://$it:18080", "iPhone hotspot") } + val discovered = services.mapValues { endpointFor(it.value, "Bonjour: ${it.key}") } + val saved = prefs.getString("pc_base_url", null)?.let { + runCatching { ConnectionPolicy.normalizeUrl(it) }.getOrNull() + ?.takeUnless { url -> ConnectionPolicy.isLoopback(url) } + ?.let { url -> endpointFor(url, "saved address") } + } + val preferred = prefs.getString("iphone_service_name", null) + val existing = endpoint + val settled = android.os.SystemClock.elapsedRealtime() - discoveryStartedAt >= 2_500 + worker.execute { + val checked = mutableMapOf() + fun healthy(candidate: ServerEndpoint?): Boolean = candidate != null && + checked.getOrPut(candidate) { health(candidate) }?.let { + ConnectionPolicy.receiverMatches(it.receiverName, preferred) + } == true + // Keep a healthy chosen endpoint; reconnecting never redirects a live session. + var chosen = existing?.takeIf { healthy(it) } + var ambiguous = false + if (chosen == null && healthy(hotspot)) chosen = hotspot + if (chosen == null && settled) { + val available = discovered.filter { (name, candidate) -> + healthy(candidate) && checked[candidate]?.receiverName == name + } + val chosenName = ConnectionPolicy.selectService(available.keys, preferred) + chosen = chosenName?.let { available[it] } + ambiguous = available.isNotEmpty() && chosen == null + } + if (chosen == null && !ambiguous && healthy(saved)) chosen = saved + val chosenName = chosen?.let { checked[it]?.receiverName } + main.post { + probing = false + if (!active) return@post + if (token != generation) { scheduleProbe(0); return@post } + if (chosenName != null) prefs.edit().putString("iphone_service_name", chosenName).apply() + publish(chosen) + if (chosen != null) { + retryDelay = 1_000L + onStatus("${chosen!!.source}: ${chosen!!.baseUrl}") + scheduleProbe(10_000) + } else { + // Resolve again after a failure: a known phone can acquire another DHCP address. + if (!resolving && resolveQueue.isEmpty()) { + resolveQueue.addAll(knownServices.values.take(8)) + resolveNext(generation) + } + onStatus(if (ambiguous) "Phone selection needed: open Connection settings" else + "Waiting for iPhone OpenGlass; join its hotspot or the same Wi-Fi") + scheduleProbe(retryDelay) + retryDelay = ConnectionPolicy.nextDelay(retryDelay) + } + } + } + } + + private data class HealthIdentity(val receiverName: String) + + private fun health(candidate: ServerEndpoint): HealthIdentity? { + var connection: HttpURLConnection? = null + return try { + connection = candidate.connection("/health").apply { + connectTimeout = 1_200 + readTimeout = 1_200 + instanceFollowRedirects = false + } + if (connection.responseCode != 200) null else { + val body = connection.inputStream.use { input -> + val data = ByteArray(4_097) + var count = 0 + val deadline = android.os.SystemClock.elapsedRealtime() + 2_500 + while (count < data.size) { + require(android.os.SystemClock.elapsedRealtime() < deadline) + val read = input.read(data, count, data.size - count) + if (read < 0) break + count += read + } + require(count <= 4_096) + String(data, 0, count, Charsets.UTF_8) + } + val json = JSONObject(body) + val receiverName = json.optString("receiver_name") + if (json.optString("service") == "openglass-ios" && json.optInt("protocol") == 1 && + ConnectionPolicy.receiverMatches(receiverName, null)) HealthIdentity(receiverName) else null + } + } catch (_: Exception) { null } finally { connection?.disconnect() } + } +} diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt index 65aad1c..2faacbd 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt @@ -2,13 +2,16 @@ package org.opensqz.openglass.rokid import android.Manifest import android.app.Activity +import android.app.AlertDialog import android.content.Context +import android.content.Intent import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.BitmapFactory import android.graphics.Color import android.graphics.ImageFormat import android.graphics.SurfaceTexture +import android.graphics.drawable.ColorDrawable import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice @@ -19,21 +22,24 @@ import android.media.AudioRecord import android.media.ImageReader import android.media.MediaRecorder import android.os.Bundle +import android.os.Build import android.os.Handler import android.os.HandlerThread import android.os.Looper +import android.text.TextUtils import android.util.Log import android.util.Range import android.util.Size import android.view.KeyEvent -import android.view.MotionEvent +import android.view.Gravity import android.view.Surface import android.view.View +import android.view.ViewGroup import android.view.WindowManager import android.widget.TextView +import android.widget.EditText import java.io.ByteArrayOutputStream import java.net.HttpURLConnection -import java.net.URL import java.nio.ByteBuffer import java.util.Locale import java.util.concurrent.atomic.AtomicBoolean @@ -57,31 +63,42 @@ private const val MANUAL_EXPOSURE_ENABLED = true private const val MANUAL_EXPOSURE_NS = 22_222_222L private const val MANUAL_FRAME_DURATION_NS = 44_444_444L private const val MANUAL_ISO = 1600 -private const val USB_REVERSE_BASE_URL = "http://127.0.0.1:18080" -private const val USB_REVERSE_WS_URL = "ws://127.0.0.1:18080/rokid/audio" private const val ENHANCE_IMAGE_FOR_UPLOAD = false private const val ENHANCE_MIN_LUMA = 55.0 private const val ENHANCE_TARGET_LUMA = 70.0 class MainActivity : Activity() { - private var pcBaseUrl = BuildConfig.PC_BASE_URL - private var pcWsUrl = BuildConfig.PC_WS_URL + private var explicitUrl: String? = null + private lateinit var endpointController: EndpointController + @Volatile private var currentEndpoint: ServerEndpoint? = null + @Volatile private var connectionStatus = "Searching for iPhone" + @Volatile private var imageConnection: HttpURLConnection? = null + @Volatile private var activeRecorder: AudioRecord? = null + private var userStopped = false + private val captureGeneration = AtomicInteger(0) + private val cameraGeneration = AtomicInteger(0) private fun configurePcAddress() { val prefs = getSharedPreferences("pc_connection", Context.MODE_PRIVATE) val supplied = intent.getStringExtra("pc_base_url") - val candidate = supplied ?: prefs.getString("pc_base_url", BuildConfig.PC_BASE_URL)!! + if (intent.getBooleanExtra("auto_discover", false)) { + explicitUrl = null + prefs.edit().putString("connection_mode", "auto").apply() + return + } + val candidate = supplied ?: when (prefs.getString("connection_mode", "auto")) { + "manual" -> prefs.getString("pc_base_url", null) + "usb" -> USB_BASE_URL + else -> null + } + if (candidate == null) { explicitUrl = null; return } try { - val uri = java.net.URI(candidate.trim().trimEnd('/')) - require(uri.scheme == "http" && !uri.host.isNullOrBlank() - && uri.rawUserInfo == null && uri.rawQuery == null && uri.rawFragment == null - && uri.rawPath.isNullOrEmpty() && (uri.port == -1 || uri.port in 1..65535)) - pcBaseUrl = uri.toString() - pcWsUrl = "ws://${uri.rawAuthority}/rokid/audio" - if (supplied != null) prefs.edit().putString("pc_base_url", pcBaseUrl).apply() - Log.i(TAG, "PC_CONFIG applied pcBase=$pcBaseUrl pcWs=$pcWsUrl") + explicitUrl = ConnectionPolicy.normalizeUrl(candidate) + if (supplied != null) prefs.edit().putString("pc_base_url", explicitUrl).apply() + Log.i(TAG, "PC_CONFIG explicit=$explicitUrl") } catch (e: Exception) { - Log.e(TAG, "Invalid PC address; keeping built-in default", e) + explicitUrl = null + Log.e(TAG, "Invalid address; using automatic discovery", e) } } @@ -111,10 +128,11 @@ class MainActivity : Activity() { private val imageUploadInFlight = AtomicBoolean(false) private val cameraRestartPending = AtomicBoolean(false) - private var audioThread: Thread? = null - private var audioWs: SimpleBinaryWebSocket? = null + @Volatile private var audioThread: Thread? = null + @Volatile private var audioWs: SimpleBinaryWebSocket? = null private var reconnectDelayMs = 1_000L - private var audioUrlIndex = 0 + private var webSocketGeneration = 0 + private val reconnectTask = Runnable { connectAudioWebSocket() } private var cameraThread: HandlerThread? = null private var cameraHandler: Handler? = null @@ -136,18 +154,45 @@ class MainActivity : Activity() { super.onCreate(savedInstanceState) configurePcAddress() window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + // Keep every layer opaque black, including the window before the HUD is drawn. + window.setBackgroundDrawable(ColorDrawable(Color.BLACK)) + window.decorView.setBackgroundColor(Color.BLACK) + window.statusBarColor = Color.BLACK + window.navigationBarColor = Color.BLACK window.decorView.systemUiVisibility = - View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY + View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or + View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY hud = TextView(this).apply { setBackgroundColor(Color.BLACK) - setTextColor(Color.rgb(40, 255, 110)) - textSize = 15f - typeface = android.graphics.Typeface.MONOSPACE - setPadding(18, 18, 18, 18) + // The full-screen clickable TextView receives D-pad focus. The platform's + // default focus highlight otherwise paints a grey rectangle over its black background. + defaultFocusHighlightEnabled = false + setTextColor(Color.rgb(140, 140, 140)) + textSize = 20f + typeface = android.graphics.Typeface.DEFAULT + gravity = Gravity.CENTER + setSingleLine(true) + ellipsize = TextUtils.TruncateAt.END + val inset = (24 * resources.displayMetrics.density).toInt() + setPadding(inset, 0, inset, 0) } - setContentView(hud) - Log.i(TAG, "onCreate pcBase=$pcBaseUrl pcWs=$pcWsUrl") + hud.setOnClickListener { resumeCapture() } + hud.setOnLongClickListener { showConnectionSettings(); true } + setContentView(hud, ViewGroup.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT, + )) + endpointController = EndpointController(this, { endpoint -> + currentEndpoint = endpoint + imageConnection?.disconnect() + closeAudioWebSocket() + reconnectDelayMs = 1_000L + if (running && endpoint != null) connectAudioWebSocket() + }, { status -> + if (connectionStatus != status) Log.i(TAG, "Connection: $status") + connectionStatus = status + }) + Log.i(TAG, "onCreate explicit=$explicitUrl") mainHandler.post(hudTick) if (hasPermissions()) { @@ -162,13 +207,14 @@ class MainActivity : Activity() { override fun onDestroy() { stopAll() + endpointController.destroy() mainHandler.removeCallbacksAndMessages(null) super.onDestroy() } override fun onResume() { super.onResume() - if (hasPermissions() && !running) { + if (hasPermissions() && !running && !userStopped) { Log.i(TAG, "onResume auto-start") startAll() } else { @@ -176,6 +222,61 @@ class MainActivity : Activity() { } } + override fun onNewIntent(newIntent: Intent) { + super.onNewIntent(newIntent) + setIntent(newIntent) + configurePcAddress() + if (running) endpointController.start(explicitUrl) + else if (hasPermissions()) { userStopped = false; startAll() } + } + + private fun showConnectionSettings() { + val prefs = getSharedPreferences("pc_connection", Context.MODE_PRIVATE) + AlertDialog.Builder(this).setTitle("OpenGlass connection") + .setItems(arrayOf("Auto: iPhone hotspot / Wi-Fi", "Manual address", "USB relay", "Choose phone again", + if (running) "Pause capture / 暂停采集" else "Resume capture / 恢复采集")) { _, which -> + fun applyMode(mode: String, url: String?) { + prefs.edit().putString("connection_mode", mode).apply() + explicitUrl = url + if (running) endpointController.start(explicitUrl) + } + when (which) { + 0 -> applyMode("auto", null) + 1 -> { + val input = EditText(this).apply { + setSingleLine(true) + setText(prefs.getString("pc_base_url", "http://172.20.10.1:18080")) + inputType = android.text.InputType.TYPE_CLASS_TEXT or android.text.InputType.TYPE_TEXT_VARIATION_URI + } + val dialog = AlertDialog.Builder(this).setTitle("iPhone / PC HTTP address") + .setView(input).setPositiveButton("Save", null).setNegativeButton("Cancel", null).create() + dialog.setOnShowListener { + dialog.getButton(AlertDialog.BUTTON_POSITIVE).setOnClickListener { + try { + val url = ConnectionPolicy.normalizeUrl(input.text.toString()) + prefs.edit().putString("pc_base_url", url).apply() + applyMode("manual", url) + dialog.dismiss() + } catch (_: Exception) { input.error = "Use http://
:18080" } + } + } + dialog.show() + } + 2 -> applyMode("usb", USB_BASE_URL) + 3 -> { + prefs.edit().remove("iphone_service_name").apply() + applyMode("auto", null) + } + 4 -> { + if (running) { + userStopped = true + stopAll() + } else resumeCapture() + } + } + }.setNegativeButton("关闭", null).show() + } + override fun onWindowFocusChanged(hasFocus: Boolean) { super.onWindowFocusChanged(hasFocus) if (hasFocus) { @@ -203,43 +304,41 @@ class MainActivity : Activity() { TAG, "key action=${event.action} code=${event.keyCode} name=${KeyEvent.keyCodeToString(event.keyCode)}", ) + if (event.keyCode == KeyEvent.KEYCODE_DPAD_CENTER || event.keyCode == KeyEvent.KEYCODE_ENTER) { + // Consume DOWN as well: forwarding it to the focused full-screen TextView can + // arm its long-click timer and unexpectedly open settings while a key is held. + if (event.action == KeyEvent.ACTION_UP && !event.isCanceled) resumeCapture() + return true + } if (event.action == KeyEvent.ACTION_UP) { when (event.keyCode) { - KeyEvent.KEYCODE_DPAD_CENTER, KeyEvent.KEYCODE_ENTER -> { - toggleStartStop() - return true - } KeyEvent.KEYCODE_BACK -> { if (running) { + userStopped = true stopAll() return true } } + KeyEvent.KEYCODE_MENU -> { showConnectionSettings(); return true } } } return super.dispatchKeyEvent(event) } - override fun dispatchTouchEvent(ev: MotionEvent): Boolean { - Log.i(TAG, "touch action=${ev.action} x=${ev.x} y=${ev.y}") - if (ev.action == MotionEvent.ACTION_UP) { - toggleStartStop() - return true - } - return super.dispatchTouchEvent(ev) - } - private fun hasPermissions(): Boolean = checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED - private fun toggleStartStop() { - if (running) stopAll() else startAll() + private fun resumeCapture() { + if (running) return + userStopped = false + if (hasPermissions()) startAll() } private fun startAll() { if (running) return running = true + captureGeneration.incrementAndGet() startMs = System.currentTimeMillis() lastError = "none" audioPackets.set(0) @@ -247,7 +346,7 @@ class MainActivity : Activity() { imagePackets.set(0) imageFailures.set(0) Log.i(TAG, "startAll") - connectAudioWebSocket() + endpointController.start(explicitUrl) startAudioCapture() scheduleCameraStart(1_200) } @@ -265,7 +364,11 @@ class MainActivity : Activity() { private fun stopAll() { if (!running) return running = false + captureGeneration.incrementAndGet() + cameraRestartPending.set(false) Log.i(TAG, "stopAll") + endpointController.stop() + imageConnection?.disconnect() stopAudioCapture() closeAudioWebSocket() stopCameraCapture() @@ -275,8 +378,10 @@ class MainActivity : Activity() { private fun connectAudioWebSocket() { if (!running) return - val urls = pcWsUrls() - val url = urls[audioUrlIndex % urls.size] + val endpoint = currentEndpoint ?: return + closeAudioWebSocket() + val token = webSocketGeneration + val url = endpoint.websocketUrl audioWsUrlText = url audioStatus = "ws connecting" Log.i(TAG, "audio WS connecting $url") @@ -284,54 +389,67 @@ class MainActivity : Activity() { url, object : SimpleBinaryWebSocket.Listener { override fun onOpen() { + mainHandler.post { + if (!running || token != webSocketGeneration) return@post wsOpen = true reconnectDelayMs = 1_000L audioStatus = "ws connected" lastError = "none" Log.i(TAG, "audio WS connected $url") + } } override fun onClosed(reason: String) { + mainHandler.post { + if (!running || token != webSocketGeneration) return@post wsOpen = false audioStatus = "ws closed" Log.w(TAG, "audio WS closed url=$url reason=$reason") scheduleReconnect() + } } override fun onFailure(error: Throwable) { + mainHandler.post { + if (!running || token != webSocketGeneration) return@post wsOpen = false audioStatus = "ws failed" - audioUrlIndex = (audioUrlIndex + 1) % urls.size lastError = "audio ws ${error.javaClass.simpleName}: ${error.message}" - Log.e(TAG, "audio WS failed url=$url; next=${urls[audioUrlIndex % urls.size]}", error) + Log.e(TAG, "audio WS failed url=$url", error) scheduleReconnect() + endpointController.retry() + } } }, + socketFactory = endpoint.network?.socketFactory, ) audioWs = ws ws.start() } - private fun pcBaseUrls(): List = - listOf(pcBaseUrl, USB_REVERSE_BASE_URL).distinct() - - private fun pcWsUrls(): List = - listOf(pcWsUrl, USB_REVERSE_WS_URL).distinct() - private fun scheduleReconnect() { if (!running) return + closeAudioWebSocket() val delay = reconnectDelayMs - reconnectDelayMs = (reconnectDelayMs * 2).coerceAtMost(5_000L) - mainHandler.postDelayed({ connectAudioWebSocket() }, delay) + reconnectDelayMs = ConnectionPolicy.nextDelay(reconnectDelayMs) + mainHandler.postDelayed(reconnectTask, delay) } private fun closeAudioWebSocket() { + mainHandler.removeCallbacks(reconnectTask) + webSocketGeneration++ wsOpen = false audioWs?.close() audioWs = null } private fun startAudioCapture() { + if (!running) return + if (audioThread?.isAlive == true) { + mainHandler.postDelayed({ startAudioCapture() }, 200) + return + } + val token = captureGeneration.get() audioThread = Thread({ val recorder = createAudioRecord() if (recorder == null) { @@ -341,10 +459,12 @@ class MainActivity : Activity() { val buf = ByteArray(CHUNK_BYTES) var lastLog = System.currentTimeMillis() try { + if (!running || token != captureGeneration.get()) return@Thread + activeRecorder = recorder recorder.startRecording() audioStatus = "recording" Log.i(TAG, "AudioRecord started") - while (running) { + while (running && token == captureGeneration.get()) { val n = recorder.read(buf, 0, buf.size) if (n <= 0) { audioDrops.incrementAndGet() @@ -366,9 +486,15 @@ class MainActivity : Activity() { } val now = System.currentTimeMillis() if (now - lastLog >= 1_000L) { + // Observe this recorder only. An unavailable configuration stays unknown; + // this diagnostic does not alter the audio source, routing or samples. + // Rokid can report true even with audible PCM, so do not infer silence from it. + val silenced = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + runCatching { recorder.activeRecordingConfiguration?.isClientSilenced }.getOrNull() + } else null Log.i( TAG, - "audio source=$audioSourceText pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nonzero=$lastNonZero wsOpen=$wsOpen", + "audio source=$audioSourceText pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nonzero=$lastNonZero wsOpen=$wsOpen silenced=${silenced?.toString() ?: "unknown"}", ) lastLog = now } @@ -382,6 +508,9 @@ class MainActivity : Activity() { } catch (_: Throwable) { } recorder.release() + if (activeRecorder === recorder) { + activeRecorder = null + } Log.i(TAG, "AudioRecord stopped") } }, "rokid-audio").also { it.start() } @@ -440,12 +569,15 @@ class MainActivity : Activity() { } private fun stopAudioCapture() { + try { activeRecorder?.stop() } catch (_: Exception) { } + audioThread?.interrupt() audioThread?.join(700) - audioThread = null + if (audioThread?.isAlive != true) audioThread = null } private fun startCameraCapture() { if (cameraThread != null || cameraDevice != null || imageReader != null) return + val token = cameraGeneration.incrementAndGet() cameraThread = HandlerThread("rokid-camera").also { it.start() } cameraHandler = Handler(cameraThread!!.looper) val manager = getSystemService(Context.CAMERA_SERVICE) as CameraManager @@ -466,7 +598,11 @@ class MainActivity : Activity() { 2, ).also { reader -> reader.setOnImageAvailableListener({ r -> - val image = r.acquireLatestImage() ?: return@setOnImageAvailableListener + val image = runCatching { r.acquireLatestImage() }.getOrNull() ?: return@setOnImageAvailableListener + if (!running || token != cameraGeneration.get()) { + image.close() + return@setOnImageAvailableListener + } try { val buffer: ByteBuffer = image.planes[0].buffer val bytes = ByteArray(buffer.remaining()) @@ -494,11 +630,13 @@ class MainActivity : Activity() { selection.cameraId, object : CameraDevice.StateCallback() { override fun onOpened(camera: CameraDevice) { + if (!running || token != cameraGeneration.get()) { camera.close(); return } cameraDevice = camera createSession(camera) } override fun onDisconnected(camera: CameraDevice) { + if (token != cameraGeneration.get()) { camera.close(); return } cameraStatus = "camera disconnected" lastError = cameraStatus Log.w(TAG, cameraStatus) @@ -507,6 +645,7 @@ class MainActivity : Activity() { } override fun onError(camera: CameraDevice, error: Int) { + if (token != cameraGeneration.get()) { camera.close(); return } cameraStatus = "camera error $error" lastError = cameraStatus Log.e(TAG, cameraStatus) @@ -514,7 +653,7 @@ class MainActivity : Activity() { restartCameraCapture(cameraStatus) } }, - cameraHandler, + mainHandler, ) } catch (t: Throwable) { cameraStatus = "camera failed" @@ -536,6 +675,7 @@ class MainActivity : Activity() { listOf(preview, jpegSurface), object : CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) { + if (!running || camera !== cameraDevice) { session.close(); return } captureSession = session cameraStatus = "camera ready $imageSizeText" Log.i(TAG, cameraStatus) @@ -544,12 +684,13 @@ class MainActivity : Activity() { } override fun onConfigureFailed(session: CameraCaptureSession) { + if (!running || camera !== cameraDevice) { session.close(); return } cameraStatus = "session configure failed" lastError = cameraStatus Log.e(TAG, cameraStatus) } }, - cameraHandler, + mainHandler, ) } catch (t: Throwable) { lastError = "session ${t.javaClass.simpleName}: ${t.message}" @@ -631,11 +772,17 @@ class MainActivity : Activity() { private fun restartCameraCapture(reason: String, delayMs: Long = 800L) { if (!running) return + val token = captureGeneration.get() + val cameraToken = cameraGeneration.get() if (!cameraRestartPending.compareAndSet(false, true)) return cameraStatus = "camera restarting" lastError = "camera restart: $reason" Log.w(TAG, "camera restart requested: $reason") mainHandler.post { + if (!running || token != captureGeneration.get() || cameraToken != cameraGeneration.get()) { + cameraRestartPending.set(false) + return@post + } try { stopCameraCapture() } catch (t: Throwable) { @@ -643,7 +790,7 @@ class MainActivity : Activity() { } mainHandler.postDelayed({ cameraRestartPending.set(false) - if (running) { + if (running && token == captureGeneration.get()) { Log.i(TAG, "camera restart now") startCameraCapture() } @@ -757,57 +904,47 @@ class MainActivity : Activity() { private fun uploadJpeg(jpeg: ByteArray) { if (!running) return - if (!imageUploadInFlight.compareAndSet(false, true)) { - imageFailures.incrementAndGet() - Log.w(TAG, "drop image upload: previous POST still in flight") - return - } + val endpoint = currentEndpoint ?: return + val token = captureGeneration.get() + if (!imageUploadInFlight.compareAndSet(false, true)) return Thread({ - val prepareStart = System.currentTimeMillis() - val downsampledJpeg = maybeDownsampleJpegForUpload(jpeg) - val postJpeg = if (ENHANCE_IMAGE_FOR_UPLOAD) enhanceJpegForModel(downsampledJpeg) else downsampledJpeg - val prepareMs = System.currentTimeMillis() - prepareStart - var posted = false - var lastFailure: String? = null - for (baseUrl in pcBaseUrls()) { - imagePostBaseText = baseUrl - try { - val postStart = System.currentTimeMillis() - val conn = (URL("$baseUrl/rokid/image").openConnection() as HttpURLConnection).apply { + var conn: HttpURLConnection? = null + try { + if (!running || token != captureGeneration.get() || endpoint != currentEndpoint) return@Thread + val downsampledJpeg = maybeDownsampleJpegForUpload(jpeg) + val postJpeg = if (ENHANCE_IMAGE_FOR_UPLOAD) enhanceJpegForModel(downsampledJpeg) else downsampledJpeg + conn = endpoint.connection("/rokid/image").apply { requestMethod = "POST" connectTimeout = 1_500 readTimeout = 4_000 + instanceFollowRedirects = false doOutput = true setRequestProperty("Content-Type", "image/jpeg") - setRequestProperty("Content-Length", postJpeg.size.toString()) + setFixedLengthStreamingMode(postJpeg.size) } + imageConnection = conn + if (!running || token != captureGeneration.get() || endpoint != currentEndpoint) return@Thread + imagePostBaseText = endpoint.baseUrl conn.outputStream.use { it.write(postJpeg) } val code = conn.responseCode - conn.disconnect() - if (code in 200..299) { + if (code !in 200..299) throw java.io.IOException("HTTP $code") + if (running && token == captureGeneration.get() && endpoint == currentEndpoint) { val count = imagePackets.incrementAndGet() cameraStatus = "image ok $imageSizeText" - Log.i( - TAG, - "image POST ok base=$baseUrl count=$count bytes=${postJpeg.size} prepareMs=$prepareMs postMs=${System.currentTimeMillis() - postStart}", - ) - posted = true - break - } else { - lastFailure = "HTTP $code at $baseUrl" - Log.e(TAG, "image POST HTTP $code base=$baseUrl") + Log.i(TAG, "image POST ok base=${endpoint.baseUrl} count=$count bytes=${postJpeg.size}") } - } catch (t: Throwable) { - lastFailure = "${t.javaClass.simpleName}: ${t.message} at $baseUrl" - Log.w(TAG, "image POST failed base=$baseUrl bytes=${postJpeg.size}", t) + } catch (t: Exception) { + if (running && token == captureGeneration.get() && endpoint == currentEndpoint) { + imageFailures.incrementAndGet() + lastError = "image post ${t.javaClass.simpleName}: ${t.message}" + cameraStatus = "image post failed" + mainHandler.post { endpointController.retry() } } + } finally { + conn?.disconnect() + if (imageConnection === conn) imageConnection = null + imageUploadInFlight.set(false) } - if (!posted) { - imageFailures.incrementAndGet() - lastError = "image post failed: $lastFailure" - cameraStatus = "image post failed" - } - imageUploadInFlight.set(false) }, "rokid-image-post").start() } @@ -900,6 +1037,7 @@ class MainActivity : Activity() { } private fun stopCameraCapture() { + cameraGeneration.incrementAndGet() cameraHandler?.removeCallbacksAndMessages(null) try { captureSession?.close() @@ -1004,20 +1142,14 @@ class MainActivity : Activity() { } private fun updateHud() { - val up = if (running) (System.currentTimeMillis() - startMs) / 1000 else 0 - val mode = if (running) "RUNNING" else "STOPPED" - hud.text = """ - OpenGlass Rokid Sensor - MODE: $mode - PC: img=$imagePostBaseText audio=$audioWsUrlText - AUDIO: $audioStatus source=$audioSourceText wsOpen=$wsOpen pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nz=$lastNonZero - IMAGE: $cameraStatus pkt=${imagePackets.get()} fail=${imageFailures.get()} size=$imageSizeText hiResDownsample=$HIGH_RES_CAPTURE_FOR_DOWNSAMPLE - UP: ${up}s - LAST ERROR: $lastError - - Enter/DPAD: start-stop - Back: stop / exit - """.trimIndent() + val status = when { + !hasPermissions() -> "请允许相机和麦克风" + !running -> "OpenGlass · 已暂停" + currentEndpoint == null || !wsOpen -> "OpenGlass · 连接中" + else -> "OpenGlass · 已连接" + } + // Keep the glasses display still; diagnostics remain in logcat. + if (hud.text.toString() != status) hud.text = status } private data class AudioStats(val rms: Double, val peak: Int, val nonZero: Int) diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt index 3ea644f..331e37f 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt @@ -11,15 +11,23 @@ import java.security.MessageDigest import java.security.SecureRandom import java.util.Locale import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean +import javax.net.SocketFactory private const val WS_TAG = "OpenGlassRokid" private const val WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" +private val WS_WATCHDOG = Executors.newSingleThreadScheduledExecutor { r -> + Thread(r, "ws-write-watchdog").apply { isDaemon = true } +} class SimpleBinaryWebSocket( private val url: String, private val listener: Listener, private val queueCapacity: Int = 8, + private val socketFactory: SocketFactory? = null, ) { interface Listener { fun onOpen() @@ -30,12 +38,15 @@ class SimpleBinaryWebSocket( private val random = SecureRandom() private val open = AtomicBoolean(false) private val stopped = AtomicBoolean(false) + private val notified = AtomicBoolean(false) private val queue = ArrayBlockingQueue(queueCapacity) private val writeLock = Object() @Volatile private var socket: Socket? = null @Volatile private var output: BufferedOutputStream? = null private var readerThread: Thread? = null private var writerThread: Thread? = null + @Volatile private var writeStartedNanos = 0L + @Volatile private var watchdog: ScheduledFuture<*>? = null fun start() { readerThread = Thread({ runReader() }, "simple-ws-reader").also { it.start() } @@ -44,15 +55,15 @@ class SimpleBinaryWebSocket( fun close() { stopped.set(true) open.set(false) - try { - writeFrame(0x8, ByteArray(0)) - } catch (_: Throwable) { - } + // Closing a stale network must never block the main thread on a socket write. try { socket?.close() } catch (_: Throwable) { } socket = null + writerThread?.interrupt() + queue.clear() + watchdog?.cancel(false) } fun isOpen(): Boolean = open.get() @@ -74,9 +85,12 @@ class SimpleBinaryWebSocket( append(if (uri.rawPath.isNullOrEmpty()) "/" else uri.rawPath) if (!uri.rawQuery.isNullOrEmpty()) append('?').append(uri.rawQuery) } - val sock = Socket() + val sock = socketFactory?.createSocket() ?: Socket() socket = sock + if (stopped.get()) return sock.connect(java.net.InetSocketAddress(uri.host, port), 3_000) + if (stopped.get()) return + sock.soTimeout = 5_000 val input = BufferedInputStream(sock.getInputStream()) val out = BufferedOutputStream(sock.getOutputStream()) output = out @@ -86,7 +100,7 @@ class SimpleBinaryWebSocket( val key = Base64.encodeToString(keyBytes, Base64.NO_WRAP) val request = ( "GET $path HTTP/1.1\r\n" + - "Host: ${uri.host}:$port\r\n" + + "Host: ${uri.rawAuthority}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key: $key\r\n" + @@ -112,7 +126,18 @@ class SimpleBinaryWebSocket( throw IOExceptionCompat("bad websocket accept") } + if (stopped.get()) return + sock.soTimeout = 0 open.set(true) + watchdog = WS_WATCHDOG.scheduleAtFixedRate({ + val started = writeStartedNanos + if (!stopped.get() && started != 0L && System.nanoTime() - started >= TimeUnit.SECONDS.toNanos(10)) { + if (notified.compareAndSet(false, true)) { + listener.onFailure(IOExceptionCompat("websocket write timed out")) + } + close() + } + }, 1, 1, TimeUnit.SECONDS) listener.onOpen() writerThread = Thread({ runWriter() }, "simple-ws-writer").also { it.start() } @@ -121,13 +146,16 @@ class SimpleBinaryWebSocket( if (opcode == 0x8) break } open.set(false) - listener.onClosed("closed") + if (!stopped.get() && notified.compareAndSet(false, true)) listener.onClosed("closed") } catch (t: Throwable) { open.set(false) - if (!stopped.get()) listener.onFailure(t) + if (!stopped.get() && notified.compareAndSet(false, true)) listener.onFailure(t) } finally { stopped.set(true) open.set(false) + writerThread?.interrupt() + queue.clear() + watchdog?.cancel(false) try { socket?.close() } catch (_: Throwable) { @@ -143,7 +171,8 @@ class SimpleBinaryWebSocket( } } catch (t: Throwable) { open.set(false) - if (!stopped.get()) listener.onFailure(t) + if (!stopped.get() && notified.compareAndSet(false, true)) listener.onFailure(t) + try { socket?.close() } catch (_: Throwable) { } } } @@ -159,9 +188,10 @@ class SimpleBinaryWebSocket( } else if (len == 127) { var longLen = 0L repeat(8) { longLen = (longLen shl 8) or readByte(input).toLong() } - if (longLen > Int.MAX_VALUE) throw IOExceptionCompat("websocket frame too large") + if (longLen < 0 || longLen > 1_048_576) throw IOExceptionCompat("websocket frame too large") len = longLen.toInt() } + if (len > 1_048_576 || (opcode >= 0x8 && len > 125)) throw IOExceptionCompat("websocket frame too large") val mask = if (masked) ByteArray(4) { readByte(input).toByte() } else null val payload = ByteArray(len) var off = 0 @@ -184,6 +214,9 @@ class SimpleBinaryWebSocket( private fun writeFrame(opcode: Int, payload: ByteArray) { val out = output ?: return synchronized(writeLock) { + if (stopped.get()) return + writeStartedNanos = System.nanoTime() + try { out.write(0x80 or (opcode and 0x0f)) val len = payload.size when { @@ -204,6 +237,9 @@ class SimpleBinaryWebSocket( out.write(mask) for (i in payload.indices) out.write(payload[i].toInt() xor mask[i % 4].toInt()) out.flush() + } finally { + writeStartedNanos = 0L + } } } diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml b/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml index 41a215d..24291ef 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml @@ -4,7 +4,15 @@ false true true + @android:color/black + @android:color/black + false + false + true + @android:color/black + @android:color/black false + false #33ff77 diff --git a/rokid_app/OpenGlassRokidSensor/app/src/test/java/org/opensqz/openglass/rokid/ConnectionPolicyTest.kt b/rokid_app/OpenGlassRokidSensor/app/src/test/java/org/opensqz/openglass/rokid/ConnectionPolicyTest.kt new file mode 100644 index 0000000..b38acb0 --- /dev/null +++ b/rokid_app/OpenGlassRokidSensor/app/src/test/java/org/opensqz/openglass/rokid/ConnectionPolicyTest.kt @@ -0,0 +1,52 @@ +package org.opensqz.openglass.rokid + +import org.junit.Assert.* +import org.junit.Test + +class ConnectionPolicyTest { + @Test fun acceptsManualIpv4AndIpv6WithoutChangingPorts() { + assertEquals("http://172.20.10.1:18080", ConnectionPolicy.normalizeUrl(" http://172.20.10.1:18080/ ")) + assertEquals("ws://[fe80::1%25wlan0]:18080/rokid/audio", + ConnectionPolicy.websocketUrl("http://[fe80::1%25wlan0]:18080")) + } + + @Test fun rejectsCredentialsPathsQueriesAndInvalidPorts() { + listOf("https://phone:18080", "http://user:pass@phone", "http://phone/path", + "http://phone?token=x", "http://phone#x", "http://phone:70000", "http://phone:0") + .forEach { value -> + assertTrue("Should reject $value", runCatching { ConnectionPolicy.normalizeUrl(value) }.isFailure) + } + } + + @Test fun loopbackStaysOffWifiBoundSockets() { + assertTrue(ConnectionPolicy.isLoopback(USB_BASE_URL)) + assertTrue(ConnectionPolicy.isLoopback("http://[::1]:18080")) + assertFalse(ConnectionPolicy.isLoopback("http://172.20.10.1:18080")) + } + + @Test fun newSelectionRequiresExactlyOnePhone() { + assertEquals("OpenGlass-one", ConnectionPolicy.selectService(setOf("OpenGlass-one"), null)) + assertNull(ConnectionPolicy.selectService(setOf("OpenGlass-one", "OpenGlass-two"), null)) + assertNull(ConnectionPolicy.selectService(emptySet(), null)) + } + + @Test fun doesNotReplaceAnUnavailablePreviouslySelectedPhone() { + assertNull(ConnectionPolicy.selectService(setOf("OpenGlass-other"), "OpenGlass-mine")) + assertEquals("OpenGlass-mine", ConnectionPolicy.selectService( + setOf("OpenGlass-other", "OpenGlass-mine"), "OpenGlass-mine")) + } + + @Test fun backoffIsBounded() { + assertEquals(2_000L, ConnectionPolicy.nextDelay(1_000)) + assertEquals(30_000L, ConnectionPolicy.nextDelay(16_000)) + assertEquals(30_000L, ConnectionPolicy.nextDelay(30_000)) + } + + @Test fun hotspotAndSavedIpMustStillMatchTheRememberedPhone() { + assertTrue(ConnectionPolicy.receiverMatches("OpenGlass-1234ABcd", null)) + assertTrue(ConnectionPolicy.receiverMatches("OpenGlass-1234ABcd", "OpenGlass-1234ABcd")) + assertFalse(ConnectionPolicy.receiverMatches("OpenGlass-1234ABcd", "OpenGlass-99999999")) + assertFalse(ConnectionPolicy.receiverMatches("", null)) + assertFalse(ConnectionPolicy.receiverMatches("some-other-service", null)) + } +} diff --git a/rokid_app/README.md b/rokid_app/README.md index 6827402..66c235b 100644 --- a/rokid_app/README.md +++ b/rokid_app/README.md @@ -2,7 +2,7 @@ [中文说明](README_zh.md) -This directory contains the adapted OpenGlass Rokid sensor app 0.1.3 and a single Android project. It captures JPEG/PCM and accepts a dynamic `pc_base_url`. Use the repository-root `glasses_panel.py` for daily startup. +This directory contains OpenGlass Rokid sensor app 0.2.0 (version code 7). It captures JPEG/PCM and can connect directly to OpenGlass on iPhone over Wi-Fi. The existing PC panel and explicit USB relay modes remain available. ## Companion setup @@ -14,25 +14,41 @@ Prepare JDK 17 (or a JDK compatible with Gradle 8.7 / AGP 8.5.2), Android SDK AP ```powershell # Build only -.\rokid_app\build_install.cmd -Sdk "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi # Build and install after connecting/authorizing USB -.\rokid_app\build_install.cmd -Sdk "" -Install -Serial "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi -Install -Serial "" ``` Use `adb devices -l` to obtain the serial. Optional `-Adb` selects an executable. The CMD delegates to `scripts/build_install.ps1`, which can also be called directly with the same options. A failed build never triggers installation. Multiple online entries require a serial; USB and wireless connections appear separately. Signature mismatches stop installation; the script never uninstalls an existing app. Allow camera/microphone permissions on the device. -Generated APK: `OpenGlassRokidSensor/app/build/outputs/apk/debug/app-debug.apk` (ignored by Git). Equivalent manual commands from the Android project root, with SDK/JDK configured: +The Wi-Fi variant is **OpenGlass WiFi**, package `org.opensqz.openglass.rokid.wifi`, version `0.2.0-wifi`. It can coexist with a collector signed on another computer, without uninstalling it or clearing its data. Run only one collector at a time. Its settings and permissions start separately. The original `-Variant Debug` (default) and release package IDs remain unchanged. + +Generated Wi-Fi APK: `OpenGlassRokidSensor/app/build/outputs/apk/wifi/app-wifi.apk` (ignored by Git). Equivalent manual commands from the Android project root, with SDK/JDK configured: ```powershell -.\gradlew.bat :app:assembleDebug +.\gradlew.bat :app:assembleWifi :app:testWifiUnitTest # Only after a successful build: -adb -s install -r .\app\build\outputs\apk\debug\app-debug.apk +adb -s install -r .\app\build\outputs\apk\wifi\app-wifi.apk ``` -## Configure and run +## iPhone hotspot or shared Wi-Fi + +Join the glasses to the iPhone hotspot using the official Rokid companion app/device settings, or join both devices to the same Wi-Fi. This app does not change Wi-Fi settings. Open iPhone OpenGlass, load its model and start its receiver, then launch this sensor app on the glasses. After camera/microphone permission is granted it starts capturing automatically. Leave both apps in the foreground. No computer or wireless ADB is needed for transport after installation. + +Default **Auto** mode checks the current Wi-Fi default gateway on port 18080 and discovers `_openglass._tcp.` with Bonjour. A saved non-loopback manual address is a fallback. Before sending sensor data, every automatic candidate must return `service=openglass-ios`, `protocol=1`, and its stable `receiver_name` from `/health`. The receiver name is remembered: moving networks may change its address, but does not silently select a different phone. Multiple unknown Bonjour receivers require manual selection/address entry. Hotel networks may isolate clients; use the iPhone hotspot if that happens. + +Auto mode never falls back to USB, and ignores loopback addresses saved by older versions. Audio and images share one selected endpoint. Reconnect delays increase from 1 second to a maximum of 30 seconds while the app remains running; network changes trigger discovery again. This is bounded backoff, not a promise to bypass a hotspot that the system disconnected. + +The default display is one dim, centered status line on black: connected, connecting, or paused. Text changes only with connection state; counters, addresses, and scrolling diagnostics stay off the display. Diagnostics remain available in logcat. Connected means the audio connection is established, not that iPhone ASR is currently ready. + +**Long-press the display or press Menu** to open settings: Auto, Manual address, USB relay, Choose phone again, and an explicit Pause/Resume capture item. Use **关闭 (Close)** or Back to dismiss the settings dialog. Manual accepts `http://
:18080`; USB selects `http://127.0.0.1:18080` and requires the existing computer USB relay. Choose phone again forgets the remembered receiver. A tap or Enter/DPAD resumes paused capture and does nothing while capture is running, so a casual touch cannot stop the stream. Holding Enter/DPAD does not open the menu. To pause, choose Pause capture from the menu. Back on the main screen still stops, then exits. The full-screen status view disables Android's default focus highlight, and the opaque window and system-bar backgrounds are black. + +On Android 10+, the audio diagnostic log includes `silenced=true/false/unknown` from this recorder's active configuration. `unknown` means unavailable, not “not silenced.” Rokid has reported `true` while capturing high-amplitude PCM, so this field is logged only and does not trigger a mute warning or recording restart. This observation does not change recording parameters or audio routing; a low RMS alone is not treated as proof of system silencing. + +Launch intents remain compatible: an explicit `pc_base_url` fixes the endpoint for that launch and persists the URL as a future fallback. `auto_discover=true` explicitly selects Auto and overrides the old saved USB address. The connection-settings mode persists, while an intent URL alone does not turn a future launcher start into USB mode. -No PC IP is needed at build time. Loopback defaults require an ADB reverse tunnel. The panel supplies and the app persists `pc_base_url` at launch. Configure the shared backend and `rokid_adb`, `rokid_mode`, `rokid_pc_url` in ignored `runtime/openglass_omni/runtime.local.json`; use the PC LAN IPv4 reachable from the glasses. Then run `python glasses_panel.py`. +## Existing PC workflow -Save a working Wi-Fi network on the glasses first. Keep USB attached until the panel establishes wireless ADB and sensor input is confirmed. Later PC/panel restarts can reuse wireless ADB. Glasses power loss may require USB recovery, but does not uninstall the app. A changed PC IP requires a local JSON edit and panel restart, not an APK rebuild. +No PC IP is needed at build time. The PC panel supplies `pc_base_url` at launch, which remains a fixed direct connection without requiring the iPhone health schema. Configure `rokid_adb`, `rokid_mode`, and `rokid_pc_url` in ignored `runtime/openglass_omni/runtime.local.json`, then run `python glasses_panel.py`. The PC workflow may use wireless ADB for launch; it is separate from the direct iPhone Wi-Fi transport above. -See [runtime setup](../runtime/openglass_omni/STARTUP_en.md) for recording and Harness prompts. +See [iPhone setup](../ios/README_zh.md) or [PC runtime setup](../runtime/openglass_omni/STARTUP_en.md). Model inference, speech recognition and multi-turn responses belong to the receiving iPhone/PC app, not this sensor collector. diff --git a/rokid_app/README_zh.md b/rokid_app/README_zh.md index a5a09ce..61fa8de 100644 --- a/rokid_app/README_zh.md +++ b/rokid_app/README_zh.md @@ -1,6 +1,6 @@ # Rokid 采集 App -此目录保存面板使用的 OpenGlass Rokid 采集 App 0.1.3：发送 JPEG 与 PCM 音频，接受面板传入的 `pc_base_url`。它不是眼镜系统固件。 +此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 8）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 ## 文件布局 @@ -8,7 +8,7 @@ - `scripts/build_install.ps1`：构建，可选安装；不启动模型或旧 bridge。 - `build_install.cmd`：同一脚本的 Windows 便捷入口。 -安装完成后，使用仓库根的 `glasses_panel.py` 启动模型与采集服务。 +安装后可以直接打开眼镜上的采集 App；电脑后端仍可使用仓库根的 `glasses_panel.py`。 ## 手机设置 @@ -21,35 +21,49 @@ 推荐在 OpenGlass 根目录运行整理后的脚本。**先只编译**： ```powershell -.\rokid_app\build_install.cmd -Sdk "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi ``` **编译并安装**：连接 USB、确认授权，用 `adb devices -l` 取得设备序列号，再运行： ```powershell -.\rokid_app\build_install.cmd -Sdk "" -Install -Serial "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi -Install -Serial "" ``` 需要指定 ADB 时增加 `-Adb ""`。也可直接调用 `scripts/build_install.ps1`，参数相同。脚本确认编译成功后才安装；设备不唯一时要求指定序列号。USB 和无线 ADB 同时在线会显示两项。 -APK 输出：`OpenGlassRokidSensor/app/build/outputs/apk/debug/app-debug.apk`。这是生成物，不提交 Git。签名不匹配时脚本会停止，**不会自动卸载**。只有确认可以清除旧采集 App 数据和权限后，才由设备所有者处理旧版本。安装后允许相机和麦克风权限。 +Wi-Fi 版显示名为 **OpenGlass WiFi**，包名 `org.opensqz.openglass.rokid.wifi`，版本 `0.2.0-wifi`。它可与其他电脑签名的旧采集 App 并存，不需要卸载或清除旧数据；同一时间只运行一个采集 App。新包的设置和权限需要单独配置。原 `-Variant Debug`（默认）与 release 的包名保留不变。 + +Wi-Fi APK 输出：`OpenGlassRokidSensor/app/build/outputs/apk/wifi/app-wifi.apk`。这是生成物，不提交 Git。签名不匹配时脚本会停止，**不会自动卸载**。安装后允许相机和麦克风权限。 ### 等价手动命令 已配置好 SDK/JDK 时，从 Android 工程目录执行： ```powershell -.\gradlew.bat :app:assembleDebug +.\gradlew.bat :app:assembleWifi :app:testWifiUnitTest # 上一步成功后再安装 -adb -s install -r .\app\build\outputs\apk\debug\app-debug.apk +adb -s install -r .\app\build\outputs\apk\wifi\app-wifi.apk ``` -## 地址配置与日常启动 +## iPhone 热点或同一个 Wi-Fi + +先通过乐奇官方 App 或眼镜网络设置，将眼镜连到 iPhone 个人热点；也可以让两台设备加入同一 Wi-Fi。采集 App 不修改 Wi-Fi 配置。打开 iPhone OpenGlass，加载模型并启动接收服务，再打开眼镜上的采集 App。相机和麦克风授权后自动开始采集。两端 App 保持前台；安装完成后，无线传输不需要电脑或无线 ADB。 + +默认 **Auto 自动模式**会探测当前 Wi-Fi 默认网关的 18080 端口，并用 Bonjour 发现 `_openglass._tcp.` 服务；保存过的非回环手动地址可作为后备。每个自动候选必须先通过 `/health` 检查：`service=openglass-ios`、`protocol=1`，以及稳定的 `receiver_name`。首次选中的手机名称会保存；换网络可以更新地址，但不会自动改连另一台手机。首次发现多台未知手机时需要明确选择地址。酒店网络可能隔离终端，无法互通时使用 iPhone 热点。 + +自动模式**不回退 USB**，会忽略旧版保存的 `127.0.0.1`。图像和音频共用一个已选地址；运行中断线按 1 秒起步、最多 30 秒的间隔重试，换网后重新发现。这不替代系统连接热点的步骤。 + +眼镜主界面采用黑底、较暗的居中单行提示：`OpenGlass · 已连接`、`OpenGlass · 连接中` 或 `OpenGlass · 已暂停`。只在状态变化时更新，不显示计数、地址或滚动调试文字；诊断信息仍可从 logcat 查看。“已连接”表示音频连接已建立，不代表 iPhone 此刻已准备好识别。 + +**长按界面或按 Menu** 打开设置，提供 Auto、Manual address、USB relay、Choose phone again，以及明确的“暂停采集 / 恢复采集”选项。点击**关闭**或按 Back 可关闭设置弹窗。手动地址格式为 `http://<地址>:18080`；USB 模式使用 `http://127.0.0.1:18080`，仍需要电脑上的既有 USB 中继。Choose phone again 会忘记已选择的手机。短按界面或 Enter/DPAD 仅用于恢复已暂停的采集，运行中不会因普通点击而停止；按住 Enter/DPAD 也不会打开菜单。需要暂停时从菜单选择“暂停采集”。主界面 Back 仍先停止，再退出。全屏状态文字关闭系统默认焦点高亮，窗口与系统栏均使用不透明黑底。 + +Android 10 及以上的音频诊断日志会读取当前 AudioRecord 的配置，记录 `silenced=true/false/unknown`；`unknown` 表示配置不可用，不能当作 `false`。实测 Rokid 在有明显音量的 PCM 输入时也可能报告 `true`，因此该字段仅保留在日志中，不用于界面静音提示或触发录音重启。诊断不会改动录音参数或声音路由，也不会仅凭 RMS 很低就断定麦克风被系统静音。 -**编译时不填写电脑 IP。** 工程默认地址是眼镜回环地址，仅用于已有 ADB reverse 的情况；面板每次启动通过 `pc_base_url` 覆盖该地址，采集 App 会保存它。 +启动参数仍兼容：显式 `pc_base_url` 会在本次启动固定连接该地址，并保存为以后自动模式的后备地址；`auto_discover=true` 显式切回自动，覆盖旧保存的 USB 地址。设置界面选择的模式会保存，仅传入一次 URL 不会把以后的普通启动永久绑定到 USB。 -安装完成后按[中文启动指南](../runtime/openglass_omni/STARTUP_zh.md)设置 `runtime.local.json`：填写后端路径、`rokid_adb`、`rokid_mode: "wifi"` 和 `rokid_pc_url: "http://:18080"`。电脑 IP 从 `ipconfig` 中选择与眼镜互通的网卡地址，不是眼镜 IP。 +## 电脑面板工作流 -先在设备/手机端保存 Wi-Fi 网络。首次保持 USB 连接，面板建立无线 ADB，确认音视频传入后拔线；以后从面板启动即可。换电脑 IP 只改本地 JSON，无需再编译 APK。眼镜断电可能重置 Wi-Fi/无线调试，需要 USB 恢复，但不会卸载 App。 +**编译时不填写电脑 IP。** 面板启动时传入的 `pc_base_url` 仍固定直连，不要求电脑提供 iPhone 的 health 字段。按[电脑中文启动指南](../runtime/openglass_omni/STARTUP_zh.md)设置 `runtime.local.json` 中的 `rokid_adb`、`rokid_mode`、`rokid_pc_url`，然后运行 `glasses_panel.py`。电脑面板的无线 ADB 启动流程与上述 iPhone 直连传输是两件事。 -详细录制与 Harness 关键词/提示词配置见启动指南。眼镜采集 App 不承担 Harness、ASR 或模型推理。 +iPhone 模型、签名安装和多轮对话见 [iPhone 中文指南](../ios/README_zh.md)。采集 App 不承担 ASR、TTS 或模型推理。 diff --git a/rokid_app/scripts/build_install.ps1 b/rokid_app/scripts/build_install.ps1 index 90d57e9..ce27a4b 100644 --- a/rokid_app/scripts/build_install.ps1 +++ b/rokid_app/scripts/build_install.ps1 @@ -2,7 +2,8 @@ param( [switch]$Install, [string]$Serial = "", [string]$Adb = $env:ROKID_ADB, - [string]$Sdk = $env:ANDROID_HOME + [string]$Sdk = $env:ANDROID_HOME, + [ValidateSet('Debug', 'Wifi')][string]$Variant = 'Debug' ) $ErrorActionPreference = "Stop" @@ -35,12 +36,13 @@ if ($Install) { Push-Location $projectRoot try { - & .\gradlew.bat :app:assembleDebug + & .\gradlew.bat ":app:assemble$Variant" if ($LASTEXITCODE -ne 0) { throw "Gradle build failed; nothing installed." } } finally { Pop-Location } -$apk = Join-Path $projectRoot "app\build\outputs\apk\debug\app-debug.apk" +$variantFolder = $Variant.ToLowerInvariant() +$apk = Join-Path $projectRoot "app\build\outputs\apk\$variantFolder\app-$variantFolder.apk" if (-not (Test-Path -LiteralPath $apk)) { throw "Build did not produce the expected APK." } Write-Host "Built: $apk" if ($Install) { @@ -50,4 +52,4 @@ if ($Install) { } Write-Host "Installed. Allow Camera and Microphone on the glasses when prompted." } -Write-Host "Configure runtime/openglass_omni/runtime.local.json, then start glasses_panel.py. No build-time PC IP is needed." +Write-Host "For iPhone: join its hotspot/shared Wi-Fi and open both OpenGlass apps (Auto mode). For PC: configure runtime.local.json and start glasses_panel.py." diff --git a/runtime/openglass_omni/rokid_device.py b/runtime/openglass_omni/rokid_device.py index b374e25..d56bfb4 100644 --- a/runtime/openglass_omni/rokid_device.py +++ b/runtime/openglass_omni/rokid_device.py @@ -291,8 +291,8 @@ def launch(self): rc, info = self.run("shell", "dumpsys", "package", self.config.get("rokid_package", "org.opensqz.openglass.rokid.debug")) - if rc or not re.search(r"versionName=0\.1\.3(?:-debug)?(?:\s|$)", info): - raise RuntimeError("尚未确认动态地址版本 0.1.3 已安装；" + if rc or not re.search(r"versionName=(?:0\.1\.3|0\.2\.0)(?:-(?:debug|wifi))?(?:\s|$)", info): + raise RuntimeError("尚未确认动态地址版本 0.1.3 / 0.2.0 已安装；" "旧版会忽略 pc_base_url，请先升级采集程序") self.run("shell", "input", "keyevent", "KEYCODE_WAKEUP") rc, out = self.run("shell", "am", "start", "-S", "-W", "-n", activity, diff --git a/tests/test_rokid_device.py b/tests/test_rokid_device.py index bfc12b1..f3743d7 100644 --- a/tests/test_rokid_device.py +++ b/tests/test_rokid_device.py @@ -30,11 +30,13 @@ def test_ambiguous_rokids_are_rejected(self): with self.assertRaises(RuntimeError): d.select() def test_launch_passes_configured_address(self): - d = self.device() - d.run = Mock(return_value=(0, "versionName=0.1.3-debug\nStatus: ok")) - d.launch() - args = d.run.call_args.args - self.assertEqual(args[-3:], ("--es", "pc_base_url", "http://192.168.1.5:18080")) + for version in ("0.1.3-debug", "0.2.0-debug", "0.2.0-wifi", "0.2.0"): + with self.subTest(version=version): + d = self.device() + d.run = Mock(return_value=(0, f"versionName={version}\nStatus: ok")) + d.launch() + args = d.run.call_args.args + self.assertEqual(args[-3:], ("--es", "pc_base_url", "http://192.168.1.5:18080")) def test_old_app_does_not_silently_ignore_new_address(self): d = self.device();d.run = Mock(return_value=(0, "versionName=0.1.2-debug"))