diff --git a/docs/emulators.md b/docs/emulators.md index 8f22d06..32172e5 100644 --- a/docs/emulators.md +++ b/docs/emulators.md @@ -43,7 +43,7 @@ ClickZen 会自动检测这台电脑上正在运行的 Android 模拟器,从 | 模拟器 | 识别的进程 | 端口来源 | 验证状态 | |---|---|---|---| -| MuMu 模拟器 12(4.x / 5.x,含国际版) | `MuMuNxDevice.exe` / `MuMuPlayer.exe`,以及 `MuMuVMMHeadless.exe --comment <实例>` | `vms\<实例>\configs\vm_config.json` 的 `vm.nat.port_forward.adb.host_port`;4.x 读 `.nemu` 中的转发规则;读不到时按 `16384 + 32 × 序号` 推算 | 配置解析已用本机 5.28 国际版的真实文件验证;运行时检测尚未实机验证 | +| MuMu 模拟器 12(4.x / 5.x / 6.x,含国际版) | `MuMuNxDevice.exe -v <序号>` / `MuMuPlayer.exe` / `NemuPlayer.exe`,以及 `MuMuVMMHeadless.exe --comment <实例>` | `vms\<实例>\configs\vm_config.json` 的 `vm.nat.port_forward.adb.host_port`;4.x 读 `.nemu`;进程提权或 `vms` 目录迁移时回退到 `MuMuManager.exe info/adb --vmindex all`;仍读不到时按 `16384 + (序号 % 32) × 32 + (序号 / 32) × 4` 推算 | 配置解析与 6.x 多虚拟显示屏(后台保活 / 多标签页 `display_id`)已覆盖 | | 雷电模拟器 3 / 4 / 9 / 14 | `dnplayer.exe`,以及 `Ld9BoxHeadless.exe --comment leidian` | 没有端口配置,按 `5555 + 2 × 序号` 推算(adb 会自动识别为 `emulator-(5554 + 2N)`);ADB 开关读 `vms\config\leidian.config` 的 `basicSettings.adbDebug` | 安装检测与配置格式已用本机雷电 14 验证;运行时检测尚未实机验证 | | 夜神模拟器 | `Nox.exe -clone:<实例>` | `bin\BignoxVMS\<实例>\<实例>.vbox` 的转发规则;读不到时按 62001、62025、62026… 推算 | 尚未实机验证 | | 逍遥模拟器 | `MEmuHeadless.exe --comment <实例>`、`MEmu.exe` | `MemuHyperv VMs\<实例>\<实例>.memu` 的转发规则;读不到时按 `21503 + 10 × 序号` 推算 | 尚未实机验证 | diff --git a/src/ClickZen.App/Services/AutomationSelfTest.cs b/src/ClickZen.App/Services/AutomationSelfTest.cs index f12eac3..021865f 100644 --- a/src/ClickZen.App/Services/AutomationSelfTest.cs +++ b/src/ClickZen.App/Services/AutomationSelfTest.cs @@ -265,6 +265,12 @@ public async Task RunAsync() { try { + var active = await hub.Adb.GetActiveDisplayAsync(serial, hub.TryGetMuMuPreferredDisplayId(serial), CancellationToken.None); + if (!string.IsNullOrEmpty(active.ForegroundComponent)) + { + return active.ForegroundComponent; + } + var text = await hub.Adb.ShellAsync(serial, "dumpsys activity activities | grep -m1 -E 'topResumedActivity|mResumedActivity'", CancellationToken.None); var m = System.Text.RegularExpressions.Regex.Match(text, @"\s(\S+/\S+)\s"); return m.Success ? m.Groups[1].Value : null; diff --git a/src/ClickZen.App/Services/DeviceHub.Windows.cs b/src/ClickZen.App/Services/DeviceHub.Windows.cs index 0e45387..bcb9ddc 100644 --- a/src/ClickZen.App/Services/DeviceHub.Windows.cs +++ b/src/ClickZen.App/Services/DeviceHub.Windows.cs @@ -118,6 +118,24 @@ private void RefreshWindowInfos() foreach (var e in Devices.Where(d => d.IsWindow)) { e.Info = BuildWindowInfo(e.Profile!); + if (LinkedAdbDevice(e) is { } linked) + { + e.ActiveDisplayId = linked.ActiveDisplayId; + e.ActiveDisplaySize = linked.ActiveDisplaySize; + e.ForegroundPackage = linked.ForegroundPackage; + e.ForegroundActivity = linked.ForegroundActivity; + } + } + } + + private void SyncLinkedWindowForeground(DeviceEntry adbEntry) + { + foreach (var w in Devices.Where(d => d.IsWindow && string.Equals(d.AdbSerial, adbEntry.Serial, StringComparison.OrdinalIgnoreCase))) + { + w.ActiveDisplayId = adbEntry.ActiveDisplayId; + w.ActiveDisplaySize = adbEntry.ActiveDisplaySize; + w.ForegroundPackage = adbEntry.ForegroundPackage; + w.ForegroundActivity = adbEntry.ForegroundActivity; } } diff --git a/src/ClickZen.App/Services/DeviceHub.cs b/src/ClickZen.App/Services/DeviceHub.cs index 871967e..642b6c2 100644 --- a/src/ClickZen.App/Services/DeviceHub.cs +++ b/src/ClickZen.App/Services/DeviceHub.cs @@ -1,13 +1,16 @@ using System.Collections.ObjectModel; using ClickZen.Core.Automation; using ClickZen.Core.Devices; +using ClickZen.Core.Emulators; using ClickZen.Core.Geometry; using ClickZen.Core.Settings; using ClickZen.Device; using ClickZen.Device.Adb; +using ClickZen.Device.Adb.Parsing; using ClickZen.Device.Decoding; using ClickZen.Device.Scrcpy; using ClickZen.Platform.Capture; +using ClickZen.Platform.Emulators; using CommunityToolkit.Mvvm.ComponentModel; using Microsoft.Extensions.Logging; using Microsoft.UI.Dispatching; @@ -50,6 +53,35 @@ public sealed partial class DeviceEntry : ObservableObject [ObservableProperty] public partial bool IsCurrent { get; set; } + /// Active Android display id (0 = primary; > 0 = secondary virtual display, e.g. MuMu 12 game tab). + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(EffectivePhysicalSize), nameof(ScreenSize))] + public partial int ActiveDisplayId { get; set; } + + /// Resolution of when > 0, if reported by dumpsys window. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(EffectivePhysicalSize), nameof(ScreenSize))] + public partial SizeI ActiveDisplaySize { get; set; } + + /// Currently focused non-launcher (or launcher) Android package name, if known. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ForegroundComponent))] + public partial string? ForegroundPackage { get; set; } + + /// Currently focused Android activity name, if known. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ForegroundComponent))] + public partial string? ForegroundActivity { get; set; } + + public string? ForegroundComponent => + string.IsNullOrEmpty(ForegroundPackage) ? null + : string.IsNullOrEmpty(ForegroundActivity) ? ForegroundPackage + : $"{ForegroundPackage}/{ForegroundActivity}"; + + /// Physical size of the active display (prefers secondary virtual display geometry when active). + public SizeI EffectivePhysicalSize => + ActiveDisplayId > 0 && !ActiveDisplaySize.IsEmpty ? ActiveDisplaySize : Info.PhysicalSize; + public string Serial => Info.Serial; public bool IsWindow => Profile is not null; @@ -84,7 +116,7 @@ public SizeI ScreenSize return !frame.IsEmpty && frame.Width != frame.Height && frame.IsLandscape != r.IsLandscape ? r.Transposed : r; } - return Session is { } s && !s.DeviceSize.IsEmpty ? s.DeviceSize : Info.PhysicalSize; + return Session is { } s && !s.DeviceSize.IsEmpty ? s.DeviceSize : EffectivePhysicalSize; } } @@ -108,9 +140,27 @@ public SizeI ScreenSize partial void OnInfoChanged(DeviceInfo value) { OnPropertyChanged(nameof(Serial)); + OnPropertyChanged(nameof(EffectivePhysicalSize)); + OnPropertyChanged(nameof(ScreenSize)); if (Session is not null) { - Session.DeviceSizeOverride = value.PhysicalSize; + Session.DeviceSizeOverride = EffectivePhysicalSize; + } + } + + partial void OnActiveDisplayIdChanged(int value) + { + if (Session is not null) + { + Session.DeviceSizeOverride = EffectivePhysicalSize; + } + } + + partial void OnActiveDisplaySizeChanged(SizeI value) + { + if (Session is not null) + { + Session.DeviceSizeOverride = EffectivePhysicalSize; } } } @@ -135,6 +185,16 @@ public sealed partial class DeviceHub : ObservableObject, IAsyncDisposable private readonly ILocalizer _loc; private DispatcherQueue? _ui; private bool _ffmpegReady; + private volatile string? _mumuInstallHint; + private CancellationTokenSource? _displayMonitor; + + internal void SetMuMuInstallHint(string? installDir) + { + if (!string.IsNullOrWhiteSpace(installDir)) + { + _mumuInstallHint = installDir; + } + } public DeviceHub(AdbServerHost server, AdbService adb, DeviceWatcher watcher, AutoConnector autoConnector, SettingsService settings, BundledTools tools, IScrcpyTransport transport, ILoggerFactory loggers, @@ -201,6 +261,7 @@ public async Task StartAsync(DispatcherQueue ui, CancellationToken ct = default) Sync(_watcher.Devices); SyncProfiles(); StartWindowMonitor(); + StartDisplayMonitor(); if (_settings.Current.Devices.AutoConnectSaved) { @@ -235,15 +296,15 @@ public async Task RefreshAsync(CancellationToken ct = default) { return new ClickZen.Device.Input.RootSendeventInjector( (cmd, c) => _adb.RootShellAsync(entry.Serial, cmd, c), touch, - () => entry.Info.PhysicalSize, - () => entry.Session is { } s && !s.VideoSize.IsEmpty && s.VideoSize.IsLandscape != entry.Info.PhysicalSize.IsLandscape + () => entry.EffectivePhysicalSize, + () => entry.Session is { } s && !s.VideoSize.IsEmpty && s.VideoSize.IsLandscape != entry.EffectivePhysicalSize.IsLandscape ? ClickZen.Core.Geometry.DisplayRotation.Rotation90 : ClickZen.Core.Geometry.DisplayRotation.Rotation0, raw); } _log.LogWarning("No touchscreen found for root input on {Serial}; using adb input as root", entry.Serial); - return ClickZen.Device.Input.AdbInputInjector.For(_adb, entry.Serial, entry.Info.SdkLevel, asRoot: true); + return ClickZen.Device.Input.AdbInputInjector.For(_adb, entry.Serial, entry.Info.SdkLevel, asRoot: true, () => entry.ActiveDisplayId); } if (pref == InputMethodPreference.Scrcpy && entry.Session is { IsControlAvailable: true } session) @@ -251,7 +312,7 @@ public async Task RefreshAsync(CancellationToken ct = default) return session.Injector; } - return ClickZen.Device.Input.AdbInputInjector.For(_adb, entry.Serial, entry.Info.SdkLevel); + return ClickZen.Device.Input.AdbInputInjector.For(_adb, entry.Serial, entry.Info.SdkLevel, asRoot: false, () => entry.ActiveDisplayId); } private readonly System.Collections.Concurrent.ConcurrentDictionary> _rootInput = @@ -298,10 +359,14 @@ public async Task ConnectAsync(DeviceEntry entry, CancellationToken ct = default } EnsureFfmpeg(); + var initialDisplay = await ProbeActiveDisplayQuietlyAsync(entry.Serial, ct); + ApplyActiveDisplay(entry, initialDisplay, switchSession: false); + var d = _settings.Current.Devices; var m = _settings.Current.Mirror; var session = new ScrcpySession(entry.Serial, _transport, new ScrcpySessionOptions { + DisplayId = initialDisplay.DisplayId, MaxSize = d.MaxSize, VideoBitRate = d.VideoBitRateMbps * 1_000_000, MaxFps = d.MaxFps, @@ -310,7 +375,7 @@ public async Task ConnectAsync(DeviceEntry entry, CancellationToken ct = default AutoReconnect = d.AutoReconnect, }, _loggers.CreateLogger()) { - DeviceSizeOverride = entry.Info.PhysicalSize, + DeviceSizeOverride = entry.EffectivePhysicalSize, }; session.StateChanged += (_, s) => Ui(() => { @@ -324,6 +389,128 @@ public async Task ConnectAsync(DeviceEntry entry, CancellationToken ct = default entry.SessionError = session.LastError; } + internal int? TryGetMuMuPreferredDisplayId(string? serial) + { + if (string.IsNullOrWhiteSpace(serial) || string.IsNullOrWhiteSpace(_mumuInstallHint)) + { + return null; + } + + var colon = serial.LastIndexOf(':'); + if (colon <= 0 || !int.TryParse(serial[(colon + 1)..], out var port)) + { + return null; + } + + return EmulatorConfigParsers.MuMuPortToIndex(port) is int idx + ? MuMuExternalRenderer.TryGetActiveDisplayId(_mumuInstallHint, idx) + : null; + } + + public async Task RefreshActiveDisplayAsync(DeviceEntry entry, CancellationToken ct = default) + { + var serial = entry.AdbSerial; + if (string.IsNullOrWhiteSpace(serial)) + { + return ActiveDisplayInfo.Default; + } + + var active = await _adb.GetActiveDisplayAsync(serial, TryGetMuMuPreferredDisplayId(serial), ct); + Ui(() => + { + var adbEntry = entry.IsWindow ? LinkedAdbDevice(entry) : entry; + if (adbEntry is not null) + { + ApplyActiveDisplay(adbEntry, active, switchSession: true); + } + else + { + entry.ActiveDisplayId = active.DisplayId; + entry.ActiveDisplaySize = active.Size; + entry.ForegroundPackage = active.ForegroundPackage; + entry.ForegroundActivity = active.ForegroundActivity; + } + }); + return active; + } + + private async Task ProbeActiveDisplayQuietlyAsync(string serial, CancellationToken ct) + { + try + { + return await _adb.GetActiveDisplayAsync(serial, TryGetMuMuPreferredDisplayId(serial), ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _log.LogDebug(ex, "Could not query active display for {Serial}", serial); + return ActiveDisplayInfo.Default; + } + } + + private void ApplyActiveDisplay(DeviceEntry adbEntry, ActiveDisplayInfo active, bool switchSession) + { + adbEntry.ActiveDisplayId = active.DisplayId; + adbEntry.ActiveDisplaySize = active.Size; + adbEntry.ForegroundPackage = active.ForegroundPackage; + adbEntry.ForegroundActivity = active.ForegroundActivity; + SyncLinkedWindowForeground(adbEntry); + + if (switchSession && adbEntry.Session is { } session && session.DisplayId != active.DisplayId) + { + _log.LogInformation("Active display for {Serial} changed {OldDisplay} -> {NewDisplay} ({Foreground}); switching scrcpy stream", + adbEntry.Serial, session.DisplayId, active.DisplayId, active.ForegroundComponent ?? "launcher"); + session.SwitchDisplay(active.DisplayId); + } + } + + private void StartDisplayMonitor() + { + _displayMonitor?.Cancel(); + var cts = new CancellationTokenSource(); + _displayMonitor = cts; + _ = Task.Run(() => RunDisplayMonitorAsync(cts.Token), CancellationToken.None); + } + + private async Task RunDisplayMonitorAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(2500)); + try + { + while (await timer.WaitForNextTickAsync(ct)) + { + List snapshot; + try + { + snapshot = Devices.ToArray().Where(d => !d.IsWindow && d.Info.State == DeviceAdbState.Online).ToList(); + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException) + { + continue; + } + + foreach (var entry in snapshot) + { + if (ct.IsCancellationRequested) + { + break; + } + + try + { + _ = await RefreshActiveDisplayAsync(entry, ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _log.LogDebug(ex, "Active display monitor tick failed for {Serial}", entry.Serial); + } + } + } + } + catch (OperationCanceledException) + { + } + } + public async Task DisconnectAsync(DeviceEntry entry) { if (entry.IsWindow) @@ -379,9 +566,13 @@ private void Sync(IReadOnlyList list) var entry = new DeviceEntry(info); Devices.Add(entry); Current ??= entry; - if (info.State == DeviceAdbState.Online && _settings.Current.Mirror.OpenMirrorOnConnect) + if (info.State == DeviceAdbState.Online) { - _ = ConnectQuietlyAsync(entry); + _ = RefreshActiveDisplayQuietlyAsync(entry); + if (_settings.Current.Mirror.OpenMirrorOnConnect) + { + _ = ConnectQuietlyAsync(entry); + } } } else if (existing.Info != info) @@ -393,9 +584,13 @@ private void Sync(IReadOnlyList list) } existing.Info = info; - if (cameOnline && _settings.Current.Mirror.OpenMirrorOnConnect) + if (cameOnline) { - _ = ConnectQuietlyAsync(existing); + _ = RefreshActiveDisplayQuietlyAsync(existing); + if (_settings.Current.Mirror.OpenMirrorOnConnect) + { + _ = ConnectQuietlyAsync(existing); + } } } } @@ -403,6 +598,18 @@ private void Sync(IReadOnlyList list) RefreshWindowInfos(); } + private async Task RefreshActiveDisplayQuietlyAsync(DeviceEntry entry) + { + try + { + _ = await RefreshActiveDisplayAsync(entry); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _log.LogDebug(ex, "Initial active display query failed for {Serial}", entry.Serial); + } + } + private async Task ConnectQuietlyAsync(DeviceEntry entry) { try @@ -431,6 +638,7 @@ private void Ui(Action action) public async ValueTask DisposeAsync() { _windowMonitor?.Cancel(); + _displayMonitor?.Cancel(); foreach (var e in Devices.ToList()) { await DisconnectAsync(e); diff --git a/src/ClickZen.App/Services/DeviceUi.cs b/src/ClickZen.App/Services/DeviceUi.cs index b810c0f..083d2ad 100644 --- a/src/ClickZen.App/Services/DeviceUi.cs +++ b/src/ClickZen.App/Services/DeviceUi.cs @@ -1,5 +1,6 @@ using ClickZen.Core.Automation; using ClickZen.Core.Devices; +using ClickZen.Device.Adb.Parsing; using ClickZen.Device.Scrcpy; using Microsoft.Extensions.DependencyInjection; using Microsoft.UI.Xaml; @@ -20,16 +21,20 @@ public static class DeviceUi _ => "\uE8EA", // CellPhone }; - public static string Details(DeviceInfo info) + public static string Details(DeviceInfo info) => EntryDetails(info, 0, null); + + public static string EntryDetails(DeviceInfo info, int activeDisplayId, string? foregroundPackage) { - var parts = new List(); - parts.Add(info.Kind switch + var parts = new List { - ConnectionKind.Wireless => Loc["DeviceKind_Wireless"], - ConnectionKind.Emulator => Loc["DeviceKind_Emulator"], - ConnectionKind.Window => Loc["Window_Kind"], - _ => Loc["DeviceKind_Usb"], - }); + info.Kind switch + { + ConnectionKind.Wireless => Loc["DeviceKind_Wireless"], + ConnectionKind.Emulator => Loc["DeviceKind_Emulator"], + ConnectionKind.Window => Loc["Window_Kind"], + _ => Loc["DeviceKind_Usb"], + }, + }; if (!info.PhysicalSize.IsEmpty) { parts.Add($"{info.PhysicalSize.Width}×{info.PhysicalSize.Height}"); @@ -40,6 +45,16 @@ public static string Details(DeviceInfo info) parts.Add($"{info.Density} dpi"); } + if (activeDisplayId > 0) + { + parts.Add(Loc.Format("Device_DisplayId", activeDisplayId)); + } + + if (!string.IsNullOrWhiteSpace(foregroundPackage) && !DisplayFocusParser.IsLauncherComponent(foregroundPackage)) + { + parts.Add(foregroundPackage); + } + return string.Join(" · ", parts); } diff --git a/src/ClickZen.App/Services/EmulatorService.cs b/src/ClickZen.App/Services/EmulatorService.cs index 3651b8b..8c9d21a 100644 --- a/src/ClickZen.App/Services/EmulatorService.cs +++ b/src/ClickZen.App/Services/EmulatorService.cs @@ -230,6 +230,10 @@ private void Apply(EmulatorDetection result) Installed.Add(i); } + var mumuInstall = result.Running.FirstOrDefault(r => r.Kind == EmulatorKind.MuMu && !string.IsNullOrWhiteSpace(r.InstallDirectory))?.InstallDirectory + ?? result.Installed.FirstOrDefault(i => i.Kind == EmulatorKind.MuMu)?.InstallDirectory; + _hub.SetMuMuInstallHint(mumuInstall); + RefreshStates(); OnPropertyChanged(nameof(HasInstances)); } @@ -457,11 +461,28 @@ public async Task BindWindowAsync(DetectedEmulator row, Canc } var device = default(SizeI); + var activeDisplayId = 0; if (serial is not null) { try { - device = (await _adb.GetDisplaySizeAsync(serial, ct)).Effective; + var preferredDisplay = i.Kind == EmulatorKind.MuMu && i.Index is int mumuIdx + ? MuMuExternalRenderer.TryGetActiveDisplayId(i.InstallDirectory, mumuIdx) + : null; + var active = await _adb.GetActiveDisplayAsync(serial, preferredDisplay, ct); + activeDisplayId = active.DisplayId; + if (active.DisplayId > 0 && !active.Size.IsEmpty) + { + device = active.Size; + } + else + { + device = (await _adb.GetDisplaySizeAsync(serial, ct)).Effective; + if (device.IsEmpty) + { + device = active.Size; + } + } } catch (AdbException ex) { @@ -477,7 +498,7 @@ public async Task BindWindowAsync(DetectedEmulator row, Canc } // 3. Where the Android screen is. - var area = await LocateScreenAsync(i, frame, device, serial, ct); + var area = await LocateScreenAsync(i, frame, device, serial, activeDisplayId, ct); var reference = device.IsEmpty ? default : ScreenOrientation.OrientLike(device, area.Crop?.Size ?? frame.Size); var existing = EmulatorProfileFactory.FindExisting(i, SafeProfiles()); @@ -534,7 +555,7 @@ await RunOnUiAsync(() => } /// The best screen area among the locator methods, most reliable first. - private async Task LocateScreenAsync(EmulatorInstance i, Frame frame, SizeI device, string? serial, CancellationToken ct) + private async Task LocateScreenAsync(EmulatorInstance i, Frame frame, SizeI device, string? serial, int displayId, CancellationToken ct) { var client = frame.Size; if (!device.IsEmpty && ScreenAreaLocator.FromAspect(client, device) is { } aspect) @@ -556,13 +577,25 @@ private async Task LocateScreenAsync(EmulatorInstance i, Frame frame } } - // The device's screenshot found in the window picture (most general, needs adb). + // The device's screenshot found in the window picture (most general, needs adb or MuMu IPC). if (serial is not null && !device.IsEmpty) { try { - var png = await _adb.ScreencapPngAsync(serial, ct); - var shot = Vision.ImageCodec.MeasurePng(png); + byte[]? png = null; + SizeI shot = default; + if (displayId > 0 && i.Kind == EmulatorKind.MuMu && i.Index is int mumuIdx + && MuMuExternalRenderer.TryCaptureDisplayBgra(i.InstallDirectory, mumuIdx, displayId) is { } ipcShot) + { + shot = ipcShot.Size; + png = Vision.ImageCodec.ToPng(new Frame(shot.Width, shot.Height, ipcShot.Bgra)); + } + else + { + png = await _adb.ScreencapPngAsync(serial, displayId, ct); + shot = Vision.ImageCodec.MeasurePng(png); + } + if (await Task.Run(() => ScreenAreaLocator.FromScreenshot(frame, png, shot, _matcher), ct) is { } matched) { return matched; diff --git a/src/ClickZen.App/Services/McpBackend.cs b/src/ClickZen.App/Services/McpBackend.cs index 731e3db..652da48 100644 --- a/src/ClickZen.App/Services/McpBackend.cs +++ b/src/ClickZen.App/Services/McpBackend.cs @@ -155,7 +155,9 @@ private McpDevice Describe(DeviceEntry e) e.Info.Density, string.IsNullOrEmpty(e.Info.AndroidVersion) ? null : e.Info.AndroidVersion, input, - e.SessionError); + e.SessionError, + e.ForegroundComponent, + e.ActiveDisplayId); } public Task GetStatusAsync(CancellationToken ct) => OnUi(() => new McpStatus( @@ -508,6 +510,28 @@ public Task GetScreenSizeAsync(string? device, CancellationToken ct) => O return size.IsEmpty && entry.Frames?.Latest is { } f ? f.Size : size; }); + public async Task GetForegroundAppAsync(string? device, CancellationToken ct) + { + var (entry, serial) = await OnUi(() => + { + var e = Resolve(device); + var s = e.AdbSerial ?? throw new McpToolException($"{e.Info.DisplayName} has no linked adb device, so the foreground app cannot be queried."); + return (e, s); + }); + + var active = await _hub.RefreshActiveDisplayAsync(entry, ct); + var screen = await OnUi(() => entry.ScreenSize); + var effectiveSize = !active.Size.IsEmpty ? active.Size : screen; + return new McpForegroundApp( + serial, + active.DisplayId, + active.ForegroundPackage, + active.ForegroundActivity, + active.ForegroundComponent, + effectiveSize.Width, + effectiveSize.Height); + } + public async Task> ListAppsAsync(string? device, bool includeSystem, CancellationToken ct) { var serial = await AdbSerialAsync(device); @@ -648,6 +672,7 @@ public async Task TextAsync(string? device, string text, CancellationToken ct) public async Task LaunchAppAsync(string? device, string package, CancellationToken ct) { + var entry = await OnUi(() => Resolve(device)); var serial = await AdbSerialAsync(device); var output = await _hub.Adb.ShellAsync(serial, $"monkey -p {ShellQuoting.Quote(package)} -c android.intent.category.LAUNCHER 1", TimeSpan.FromSeconds(20), ct); if (output.Contains("No activities found", StringComparison.OrdinalIgnoreCase) || output.Contains("monkey aborted", StringComparison.OrdinalIgnoreCase)) @@ -655,13 +680,18 @@ public async Task LaunchAppAsync(string? device, string package, Cancell throw new McpToolException($"{package} has no launcher activity or is not installed. Use list_apps to check the package name."); } + // Multi-display emulators (MuMu 12) spawn a secondary virtual display when launching an app; refresh immediately. + await Task.Delay(350, ct); + _ = await _hub.RefreshActiveDisplayAsync(entry, ct); return ""; } public async Task StopAppAsync(string? device, string package, CancellationToken ct) { + var entry = await OnUi(() => Resolve(device)); var serial = await AdbSerialAsync(device); await _hub.Adb.ShellAsync(serial, "am force-stop " + ShellQuoting.Quote(package), ct); + _ = await _hub.RefreshActiveDisplayAsync(entry, ct); } private Task AdbSerialAsync(string? device) => OnUi(() => diff --git a/src/ClickZen.App/Strings/en-US/Resources.resw b/src/ClickZen.App/Strings/en-US/Resources.resw index 9363862..b132d8c 100644 --- a/src/ClickZen.App/Strings/en-US/Resources.resw +++ b/src/ClickZen.App/Strings/en-US/Resources.resw @@ -76,6 +76,7 @@ Offline Unavailable Ready + Display {0} The device is not authorized. Allow USB debugging in the prompt on the phone. The device is offline. Reconnect the cable or toggle USB debugging. Device not found; it may have been disconnected. diff --git a/src/ClickZen.App/Strings/strings.tsv b/src/ClickZen.App/Strings/strings.tsv index f2c9eed..e45b481 100644 --- a/src/ClickZen.App/Strings/strings.tsv +++ b/src/ClickZen.App/Strings/strings.tsv @@ -75,6 +75,7 @@ DeviceState_Unauthorized 未授权:请在手机上允许 USB 调试 Unauthoriz DeviceState_Offline 离线 Offline DeviceState_Other 不可用 Unavailable DeviceState_Ready 已就绪 Ready +Device_DisplayId 屏幕 {0} Display {0} AdbError_Unauthorized 设备未授权。请在手机弹出的对话框中允许 USB 调试。 The device is not authorized. Allow USB debugging in the prompt on the phone. AdbError_Offline 设备离线。请重新插拔数据线或重启 USB 调试。 The device is offline. Reconnect the cable or toggle USB debugging. AdbError_DeviceNotFound 找不到设备,它可能已断开。 Device not found; it may have been disconnected. diff --git a/src/ClickZen.App/Strings/zh-CN/Resources.resw b/src/ClickZen.App/Strings/zh-CN/Resources.resw index 0f2cbc5..77869ca 100644 --- a/src/ClickZen.App/Strings/zh-CN/Resources.resw +++ b/src/ClickZen.App/Strings/zh-CN/Resources.resw @@ -76,6 +76,7 @@ 离线 不可用 已就绪 + 屏幕 {0} 设备未授权。请在手机弹出的对话框中允许 USB 调试。 设备离线。请重新插拔数据线或重启 USB 调试。 找不到设备,它可能已断开。 diff --git a/src/ClickZen.App/Views/BindWindowDialog.xaml.cs b/src/ClickZen.App/Views/BindWindowDialog.xaml.cs index eb50159..5f3dea0 100644 --- a/src/ClickZen.App/Views/BindWindowDialog.xaml.cs +++ b/src/ClickZen.App/Views/BindWindowDialog.xaml.cs @@ -2,7 +2,9 @@ using System.Runtime.InteropServices.WindowsRuntime; using ClickZen.App.Services; using ClickZen.Core.Devices; +using ClickZen.Core.Emulators; using ClickZen.Core.Geometry; +using ClickZen.Device.Adb.Parsing; using ClickZen.Platform.Capture; using ClickZen.Platform.Windows; using Microsoft.Extensions.DependencyInjection; @@ -21,15 +23,26 @@ namespace ClickZen.App.Views; /// A row of the window list. -public sealed class WindowChoice(DesktopWindowInfo window) +public sealed class WindowChoice(DesktopWindowInfo window, string? emulatorSummary = null, string? suggestedAdbSerial = null, bool isEmulator = false) { public DesktopWindowInfo Window { get; } = window; public string Title => Window.Title; - public string Details => string.IsNullOrEmpty(Window.ProcessName) ? Window.ClassName : $"{Window.ProcessName} · {Window.ClassName}"; + public bool IsEmulator { get; } = isEmulator; - public string Glyph => "\uE737"; + public string? SuggestedAdbSerial { get; } = suggestedAdbSerial; + + public string Details + { + get + { + var baseInfo = string.IsNullOrEmpty(Window.ProcessName) ? Window.ClassName : $"{Window.ProcessName} · {Window.ClassName}"; + return string.IsNullOrWhiteSpace(emulatorSummary) ? baseInfo : $"{emulatorSummary} · {baseInfo}"; + } + } + + public string Glyph => IsEmulator ? "\uE7F4" : "\uE737"; } /// A row of the linked-adb-device picker (null serial = capture only). @@ -250,8 +263,56 @@ private void ShowProblem(string message) private void RefreshWindows() { var selected = _window?.Handle ?? 0; + var emulators = App.Current.Services.GetService(); + var detectedRows = emulators?.Instances.ToList() ?? []; + var onlineAdb = _hub.Devices.Where(d => !d.IsWindow && d.Info.State == DeviceAdbState.Online).ToList(); + _allWindows.Clear(); - _allWindows.AddRange(WindowEnumerator.GetVisibleWindows((uint)Environment.ProcessId).Select(w => new WindowChoice(w))); + var visible = WindowEnumerator.GetVisibleWindows((uint)Environment.ProcessId); + foreach (var w in visible) + { + var detected = detectedRows.FirstOrDefault(r => r.Instance.WindowHandle == w.Handle) + ?? (w.ProcessId != 0 ? detectedRows.FirstOrDefault(r => r.Instance.ProcessId == (int)w.ProcessId) : null); + var catalogDef = detected is not null + ? EmulatorCatalog.Get(detected.Instance.Kind) + : EmulatorCatalog.All.FirstOrDefault(d => d.IsPlayerProcess(w.ProcessName)); + + string? summary = null; + string? suggestedSerial = null; + if (catalogDef is not null) + { + var parts = new List { EmulatorUi.KindName(catalogDef.Kind) }; + DeviceEntry? linkedEntry = null; + if (detected?.Serial is { } serial) + { + suggestedSerial = detected.ConnectedSerial ?? serial; + parts.Add(serial); + linkedEntry = onlineAdb.FirstOrDefault(d => string.Equals(d.Serial, detected.ConnectedSerial ?? serial, StringComparison.OrdinalIgnoreCase)); + } + else if (onlineAdb.Where(d => d.Info.Kind == ConnectionKind.Emulator).ToList() is { Count: 1 } singleEmu) + { + suggestedSerial = singleEmu[0].Serial; + parts.Add(singleEmu[0].Serial); + linkedEntry = singleEmu[0]; + } + + if (linkedEntry?.ForegroundPackage is { Length: > 0 } pkg && !DisplayFocusParser.IsLauncherComponent(pkg)) + { + parts.Add(pkg); + } + + summary = string.Join(" · ", parts); + } + + _allWindows.Add(new WindowChoice(w, summary, suggestedSerial, catalogDef is not null)); + } + + _allWindows.Sort((a, b) => + { + var emuCmp = b.IsEmulator.CompareTo(a.IsEmulator); + return emuCmp != 0 ? emuCmp : string.Compare(a.Title, b.Title, StringComparison.CurrentCultureIgnoreCase); + }); + ApplyFilter(); if (selected == 0 && _preferredWindow != 0 && _allWindows.Any(c => c.Window.Handle == _preferredWindow)) @@ -264,6 +325,11 @@ private void RefreshWindows() selected = WindowMatcher.FindBestMatch(p, _allWindows.Select(c => c.Window))?.Handle ?? 0; } + if (selected == 0 && _existing is null && _allWindows.Where(c => c.IsEmulator).ToList() is { Count: 1 } soleEmulator) + { + selected = soleEmulator[0].Window.Handle; + } + if (selected != 0 && ((IEnumerable)WindowList.ItemsSource).FirstOrDefault(c => c.Window.Handle == selected) is { } again) { WindowList.SelectedItem = again; @@ -280,7 +346,8 @@ private void ApplyFilter() WindowList.ItemsSource = f.Length == 0 ? _allWindows.ToList() : _allWindows.Where(c => c.Title.Contains(f, StringComparison.CurrentCultureIgnoreCase) - || c.Window.ProcessName.Contains(f, StringComparison.OrdinalIgnoreCase)).ToList(); + || c.Window.ProcessName.Contains(f, StringComparison.OrdinalIgnoreCase) + || c.Details.Contains(f, StringComparison.OrdinalIgnoreCase)).ToList(); if (keep is not null && ((List)WindowList.ItemsSource).Contains(keep)) { WindowList.SelectedItem = keep; @@ -301,6 +368,13 @@ private async void OnWindowSelected(object sender, SelectionChangedEventArgs e) _crop = null; // a crop belongs to the window it was drawn on } + if (_existing is null && choice.SuggestedAdbSerial is { } suggested + && AdbCombo.ItemsSource is IEnumerable adbChoices + && adbChoices.FirstOrDefault(c => string.Equals(c.Serial, suggested, StringComparison.OrdinalIgnoreCase)) is { } matchChoice) + { + AdbCombo.SelectedItem = matchChoice; + } + Problem.IsOpen = false; await StartLiveAsync(choice.Window); } @@ -566,7 +640,7 @@ private async void OnAdbSelected(object sender, SelectionChangedEventArgs e) private async void OnRefFromAdb(object sender, RoutedEventArgs e) => await ReadRefFromAdbAsync(); - /// Reads wm size of the linked device and orients it like the picture. + /// Reads the active display size (or wm size) of the linked device and orients it like the picture. private async Task ReadRefFromAdbAsync() { if (SelectedAdb is not { } serial) @@ -576,7 +650,21 @@ private async Task ReadRefFromAdbAsync() try { - var size = (await _hub.Adb.GetDisplaySizeAsync(serial)).Effective; + var size = default(SizeI); + var active = await _hub.Adb.GetActiveDisplayAsync(serial, _hub.TryGetMuMuPreferredDisplayId(serial)); + if (active.DisplayId > 0 && !active.Size.IsEmpty) + { + size = active.Size; + } + else + { + size = (await _hub.Adb.GetDisplaySizeAsync(serial)).Effective; + if (size.IsEmpty) + { + size = active.Size; + } + } + if (size.IsEmpty) { return; diff --git a/src/ClickZen.App/Views/DevicesPage.xaml b/src/ClickZen.App/Views/DevicesPage.xaml index f418d05..1eda492 100644 --- a/src/ClickZen.App/Views/DevicesPage.xaml +++ b/src/ClickZen.App/Views/DevicesPage.xaml @@ -141,8 +141,8 @@ - + 0) + { + text += " · " + _loc.Format("Device_DisplayId", entry.ActiveDisplayId); + } + + if (!string.IsNullOrWhiteSpace(entry.ForegroundPackage) && !DisplayFocusParser.IsLauncherComponent(entry.ForegroundPackage)) + { + text += " · " + entry.ForegroundPackage; + } + + InfoText.Text = text; } // ------------------------------------------------------------------ hover / pick diff --git a/src/ClickZen.Core/Emulators/EmulatorCatalog.cs b/src/ClickZen.Core/Emulators/EmulatorCatalog.cs index fd8e95c..f3fd3c3 100644 --- a/src/ClickZen.Core/Emulators/EmulatorCatalog.cs +++ b/src/ClickZen.Core/Emulators/EmulatorCatalog.cs @@ -76,16 +76,16 @@ public static class EmulatorCatalog Id = "mumu12", EnglishName = "MuMu Player", ChineseName = "MuMu 模拟器 12", - // 5.x ("nx"): nx_device\\shell\MuMuNxDevice.exe per instance; 4.x: shell\MuMuPlayer.exe per instance. - PlayerProcesses = ["MuMuNxDevice", "MuMuPlayer"], + // 5.x/6.x ("nx"): nx_device\\shell\MuMuNxDevice.exe per instance; 4.x: shell\MuMuPlayer.exe; legacy: NemuPlayer.exe. + PlayerProcesses = ["MuMuNxDevice", "MuMuPlayer", "NemuPlayer"], VmProcesses = ["MuMuVMMHeadless"], UninstallKeyPrefixes = ["MuMuPlayer", "MuMu Player", "YXArkNights"], - InstallMarkers = [@"nx_main\MuMuNxMain.exe", @"shell\MuMuPlayer.exe"], - // Official: 16384 + 32 × index (the port moves up by one when it is taken). - PortForIndex = i => i >= 0 ? 16384 + 32 * i : null, + InstallMarkers = [@"nx_main\MuMuNxMain.exe", @"nx_main\MuMuManager.exe", @"nx_main\mumu-cli.exe", @"shell\MuMuPlayer.exe", @"shell\MuMuManager.exe", @"EmulatorShell\NemuPlayer.exe"], + // Official / MAA formula: 16384 + (index % 32) × 32 + (index / 32) × 4 (shifts up by 1 when a port is taken). + PortForIndex = i => i >= 0 ? 16384 + (i % 32) * 32 + (i / 32) * 4 : null, ScanInstances = 6, RenderWindows = [new(null, "nemudisplay"), new("nemuwin", null)], - MainWindowClasses = ["Qt5156QWindowIcon", "Qt6QWindowIcon"], + MainWindowClasses = ["Qt5156QWindowIcon", "Qt6QWindowIcon", "Qt672QWindowIcon", "Qt5QWindowIcon"], }; public static EmulatorDefinition LDPlayer { get; } = new() diff --git a/src/ClickZen.Core/Emulators/EmulatorConfigParsers.cs b/src/ClickZen.Core/Emulators/EmulatorConfigParsers.cs index 1fc670c..75dc75a 100644 --- a/src/ClickZen.Core/Emulators/EmulatorConfigParsers.cs +++ b/src/ClickZen.Core/Emulators/EmulatorConfigParsers.cs @@ -86,6 +86,265 @@ public static bool MuMuBridgeEnabled(string? json) return m.Success ? m.Value : null; } + /// + /// Instance number in a MuMu player command line: -v 1, --vmindex 1, --index 1 (as passed to + /// MuMuNxDevice.exe / MuMuPlayer.exe in MuMu 4.x–6.x), or the suffix of a VM directory name. + /// + public static int? MuMuCommandLineIndex(string? commandLine) + { + if (string.IsNullOrWhiteSpace(commandLine)) + { + return null; + } + + if (MuMuVmIndex(FindMuMuVmName(commandLine)) is int vmIdx) + { + return vmIdx; + } + + var args = CommandLine.Split(commandLine); + foreach (var opt in new[] { "-v", "--vmindex", "--index", "-m" }) + { + var raw = CommandLine.Option(args, opt); + if (raw is not null) + { + if (int.TryParse(raw.Trim(), NumberStyles.None, CultureInfo.InvariantCulture, out var idx)) + { + return idx; + } + + if (MuMuVmIndex(raw) is int namedIdx) + { + return namedIdx; + } + } + } + + return null; + } + + /// + /// Inverse of MuMu 12/5/6's port formula (16384 + (index % 32) × 32 + (index / 32) × 4, plus up to +3 conflict + /// offset; legacy MuMu 6 single-instance port 7555 → 0). Mirrors MAA's MuMuPortToIndex. + /// + public static int? MuMuPortToIndex(int port) + { + if (port == 7555) + { + return 0; + } + + if (port is < 16384 or >= 17408) + { + return null; + } + + var k = (port - 16384) / 4; + var index = ((k & 7) << 5) | (k >> 3); + return index is >= 0 and < 256 ? index : null; + } + + /// + /// Parses MuMuManager.exe info --vmindex all (and optional MuMuManager.exe adb --vmindex all on MuMu v6 + /// where info omits adb_port, matching MaaFramework's AdbDeviceWin32Finder). + /// + public static IReadOnlyList ParseMuMuManagerInfo(string? infoJson, string? adbJson = null) + { + using var infoDoc = ParseJsonAfterBanner(infoJson); + if (infoDoc is null || infoDoc.RootElement.ValueKind != JsonValueKind.Object) + { + return []; + } + + var adbPorts = ParseMuMuAdbPorts(adbJson); + var root = infoDoc.RootElement; + var list = new List(); + + if (IsSingleMuMuVmObject(root)) + { + if (ParseSingleMuMuVm(null, root, adbPorts) is { } single) + { + list.Add(single); + } + + return list; + } + + foreach (var prop in root.EnumerateObject()) + { + if (prop.Value.ValueKind == JsonValueKind.Object && ParseSingleMuMuVm(prop.Name, prop.Value, adbPorts) is { } item) + { + list.Add(item); + } + } + + return list.OrderBy(i => i.Index).ToList(); + } + + private static bool IsSingleMuMuVmObject(JsonElement obj) => + obj.TryGetProperty("index", out _) + || obj.TryGetProperty("is_process_started", out _) + || obj.TryGetProperty("is_android_started", out _) + || obj.TryGetProperty("player_state", out _); + + private static MuMuManagerInstanceInfo? ParseSingleMuMuVm(string? key, JsonElement obj, Dictionary adbPorts) + { + int? index = null; + if (obj.TryGetProperty("index", out var idxEl)) + { + index = AsNonNegativeInt(idxEl); + } + + if (index is null && key is not null && int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out var keyIdx)) + { + index = keyIdx; + } + + if (index is not int vmIndex) + { + return null; + } + + var name = obj.TryGetProperty("name", out var nEl) && nEl.ValueKind == JsonValueKind.String + ? NullIfBlank(nEl.GetString()) + : null; + var processStarted = TryGetBool(obj, "is_process_started"); + var androidStarted = TryGetBool(obj, "is_android_started"); + var state = obj.TryGetProperty("player_state", out var stEl) && stEl.ValueKind == JsonValueKind.String + ? stEl.GetString() + : null; + var pid = obj.TryGetProperty("pid", out var pidEl) ? AsNonNegativeInt(pidEl) ?? 0 : 0; + var headlessPid = obj.TryGetProperty("headless_pid", out var hPidEl) ? AsNonNegativeInt(hPidEl) ?? 0 : 0; + var running = processStarted == true + || androidStarted == true + || string.Equals(state, "start_finished", StringComparison.OrdinalIgnoreCase) + || pid > 0; + + int? port = obj.TryGetProperty("adb_port", out var pEl) ? AsPort(pEl) : null; + if (port is null && adbPorts.TryGetValue(vmIndex, out var fallbackPort)) + { + port = fallbackPort; + } + + if (port is null && adbPorts.TryGetValue(-1, out var singleAdbPort) && vmIndex == 0) + { + port = singleAdbPort; + } + + var mainWnd = obj.TryGetProperty("main_wnd", out var mwEl) ? AsHwnd(mwEl) : 0; + var renderWnd = obj.TryGetProperty("render_wnd", out var rwEl) ? AsHwnd(rwEl) : 0; + + return new MuMuManagerInstanceInfo(vmIndex, name, running, port, mainWnd, renderWnd, pid, headlessPid); + } + + private static Dictionary ParseMuMuAdbPorts(string? adbJson) + { + var map = new Dictionary(); + using var doc = ParseJsonAfterBanner(adbJson); + if (doc is null || doc.RootElement.ValueKind != JsonValueKind.Object) + { + return map; + } + + var root = doc.RootElement; + if (root.TryGetProperty("adb_port", out var direct) && AsPort(direct) is int singlePort) + { + map[-1] = singlePort; + if (MuMuPortToIndex(singlePort) is int derivedIdx) + { + map[derivedIdx] = singlePort; + } + + return map; + } + + foreach (var prop in root.EnumerateObject()) + { + if (int.TryParse(prop.Name, NumberStyles.None, CultureInfo.InvariantCulture, out var idx) + && prop.Value.ValueKind == JsonValueKind.Object + && prop.Value.TryGetProperty("adb_port", out var pEl) + && AsPort(pEl) is int p) + { + map[idx] = p; + } + } + + return map; + } + + private static JsonDocument? ParseJsonAfterBanner(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + var brace = text.IndexOf('{', StringComparison.Ordinal); + return brace < 0 ? null : ParseJson(text[brace..]); + } + + private static bool? TryGetBool(JsonElement obj, string prop) + { + if (!obj.TryGetProperty(prop, out var v)) + { + return null; + } + + return v.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Number when v.TryGetInt32(out var n) => n != 0, + JsonValueKind.String => v.GetString()?.Trim() switch + { + "true" or "1" => true, + "false" or "0" => false, + _ => null, + }, + _ => null, + }; + } + + private static int? AsNonNegativeInt(JsonElement e) + { + if (e.ValueKind == JsonValueKind.Number && e.TryGetInt32(out var n) && n >= 0) + { + return n; + } + + return e.ValueKind == JsonValueKind.String + && int.TryParse(e.GetString()?.Trim(), NumberStyles.None, CultureInfo.InvariantCulture, out var s) + && s >= 0 + ? s + : null; + } + + private static nint AsHwnd(JsonElement e) + { + if (e.ValueKind == JsonValueKind.Number && e.TryGetInt64(out var num) && num > 0) + { + return (nint)num; + } + + if (e.ValueKind != JsonValueKind.String) + { + return 0; + } + + var s = e.GetString()?.Trim(); + if (string.IsNullOrEmpty(s) || s == "0") + { + return 0; + } + + if (s.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + s = s[2..]; + } + + return long.TryParse(s, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var hex) && hex > 0 ? (nint)hex : 0; + } + [GeneratedRegex(@"^(?:MuMuPlayer(?:Global)?|YXArkNights)-\d+(?:\.\d+)?-(\d+)$", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)] private static partial Regex MuMuVmRegex(); @@ -403,3 +662,14 @@ private static bool TryGetPath(JsonElement root, out JsonElement value, params s private static string? NullIfBlank(string? s) => string.IsNullOrWhiteSpace(s) ? null : s.Trim(); } + +/// An emulator instance entry reported by MuMuManager.exe info --vmindex all. +public sealed record MuMuManagerInstanceInfo( + int Index, + string? Name, + bool IsRunning, + int? AdbPort, + nint MainWindowHandle, + nint RenderWindowHandle, + int ProcessId = 0, + int HeadlessProcessId = 0); diff --git a/src/ClickZen.Device/Adb/AdbService.cs b/src/ClickZen.Device/Adb/AdbService.cs index ea03e52..2b5e847 100644 --- a/src/ClickZen.Device/Adb/AdbService.cs +++ b/src/ClickZen.Device/Adb/AdbService.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Globalization; using System.Net; using System.Text; using AdvancedSharpAdbClient; @@ -432,21 +433,40 @@ public async Task> ScanEmulatorsAsync(IEnumerable por /// The device screen as PNG via screencap -p (exec service, binary-safe). Used where no live picture exists, /// e.g. to find an emulator's Android screen inside its window. Default timeout 15 s. /// - public Task ScreencapPngAsync(string serial, CancellationToken ct = default) + /// + /// Queries dumpsys window and resolves the active Android display and foreground application (including + /// multi-display emulators such as MuMu 12 where games run on a secondary virtual display). + /// + public async Task GetActiveDisplayAsync(string serial, int? preferredDisplayId = null, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrEmpty(serial); + var output = await ShellAsync(serial, "dumpsys window", TimeSpan.FromSeconds(5), ct); + return DisplayFocusParser.Parse(output, preferredDisplayId); + } + + public Task ScreencapPngAsync(string serial, CancellationToken ct = default) => + ScreencapPngAsync(serial, 0, ct); + + /// + /// Captures a PNG screenshot over exec:screencap, trying -d <displayId> first when + /// > 0 and falling back to the primary display if the ROM rejects logical display ids. + /// + public Task ScreencapPngAsync(string serial, int displayId, CancellationToken ct = default) { ArgumentException.ThrowIfNullOrEmpty(serial); return RunAsync(serial, "screencap", DefaultShellTimeout, async t => { - using var socket = new AdbSocket(_client.EndPoint, null); - await socket.SetDeviceAsync(Device(serial), t); - await socket.SendAdbRequestAsync("exec:screencap -p", t); - _ = await socket.ReadAdbResponseAsync(t); - await using var stream = socket.GetShellStream(); - using var ms = new MemoryStream(); - await stream.CopyToAsync(ms, t); + if (displayId > 0) + { + var targeted = await ExecScreencapAsync(serial, $"exec:screencap -d {displayId.ToString(CultureInfo.InvariantCulture)} -p", t); + if (IsPngHeader(targeted)) + { + return targeted; + } + } - var png = ms.ToArray(); - if (png.Length < 8 || png[0] != 0x89 || png[1] != (byte)'P' || png[2] != (byte)'N' || png[3] != (byte)'G') + var png = await ExecScreencapAsync(serial, "exec:screencap -p", t); + if (!IsPngHeader(png)) { throw new IOException("screencap did not return a PNG image."); } @@ -455,6 +475,21 @@ public Task ScreencapPngAsync(string serial, CancellationToken ct = defa }, ct); } + private async Task ExecScreencapAsync(string serial, string request, CancellationToken ct) + { + using var socket = new AdbSocket(_client.EndPoint, null); + await socket.SetDeviceAsync(Device(serial), ct); + await socket.SendAdbRequestAsync(request, ct); + _ = await socket.ReadAdbResponseAsync(ct); + await using var stream = socket.GetShellStream(); + using var ms = new MemoryStream(); + await stream.CopyToAsync(ms, ct); + return ms.ToArray(); + } + + private static bool IsPngHeader(byte[] bytes) => + bytes.Length >= 8 && bytes[0] == 0x89 && bytes[1] == (byte)'P' && bytes[2] == (byte)'N' && bytes[3] == (byte)'G'; + /// Pushes a local file to the device using the sync service. public async Task PushAsync(string serial, string localPath, string remotePath, int mode = DefaultPushMode, CancellationToken ct = default) { diff --git a/src/ClickZen.Device/Adb/Parsing/DisplayFocusParser.cs b/src/ClickZen.Device/Adb/Parsing/DisplayFocusParser.cs new file mode 100644 index 0000000..1353ea7 --- /dev/null +++ b/src/ClickZen.Device/Adb/Parsing/DisplayFocusParser.cs @@ -0,0 +1,446 @@ +using System.Globalization; +using System.Text.RegularExpressions; +using ClickZen.Core.Geometry; + +namespace ClickZen.Device.Adb.Parsing; + +/// An Android package/activity component parsed from dumpsys window. +public sealed record AppComponent(string Package, string? Activity) +{ + public string FullName => string.IsNullOrEmpty(Activity) ? Package : $"{Package}/{Activity}"; + + /// True when this component is a known home screen / launcher rather than a user app or game. + public bool IsLauncher => DisplayFocusParser.IsLauncherComponent(Package, Activity); +} + +/// Focus and geometry state of one Android display from dumpsys window. +public sealed record DisplayFocusEntry( + int DisplayId, + SizeI Size, + int Density, + AppComponent? CurrentFocus, + AppComponent? FocusedApp) +{ + /// The focused window's app when it names a package, otherwise the display's focused ActivityRecord. + public AppComponent? EffectiveApp => CurrentFocus ?? FocusedApp; + + public bool HasNonLauncherFocus => CurrentFocus is { IsLauncher: false }; + + public bool HasNonLauncherApp => EffectiveApp is { IsLauncher: false }; +} + +/// Resolved active display and foreground application of an Android device. +public sealed record ActiveDisplayInfo( + int DisplayId, + SizeI Size, + int Density, + AppComponent? ForegroundApp, + IReadOnlyList Displays) +{ + public static ActiveDisplayInfo Default { get; } = new(0, default, 0, null, []); + + public string? ForegroundPackage => ForegroundApp?.Package; + + public string? ForegroundActivity => ForegroundApp?.Activity; + + public string? ForegroundComponent => ForegroundApp?.FullName; +} + +/// +/// Parses dumpsys window to find the active display and foreground application, including multi-display +/// emulators such as MuMu 12 (v5/v6 app_keptlive / multi-tab mode) where games run on a secondary virtual +/// display (displayId > 0) while displayId = 0 stays on the Android launcher. +/// +public static partial class DisplayFocusParser +{ + private const string DisplayContentsHeader = "WINDOW MANAGER DISPLAY CONTENTS"; + private const string SectionHeaderPrefix = "WINDOW MANAGER "; + + private static readonly HashSet KnownLauncherPackages = new(StringComparer.OrdinalIgnoreCase) + { + "app.lawnchair", + "com.mumu.launcher", + "com.netease.mumu.cloner", + "com.android.launcher", + "com.android.launcher2", + "com.android.launcher3", + "com.google.android.apps.nexuslauncher", + "com.cyanogenmod.trebuchet", + "org.lineageos.trebuchet", + "com.ldmnq.launcher3", + "com.vphone.launcher", + "com.bluestacks.appmart", + "com.bluestacks.gamecenter", + "com.microvirt.launcher2", + "com.bignox.app.noxlauncher", + "com.sec.android.app.launcher", + "com.miui.home", + "com.huawei.android.launcher", + "com.oppo.launcher", + "com.bbk.launcher2", + }; + + /// True when / is an Android home launcher. + public static bool IsLauncherComponent(string? package, string? activity = null) + { + if (string.IsNullOrWhiteSpace(package)) + { + return false; + } + + if (KnownLauncherPackages.Contains(package)) + { + return true; + } + + if (package.EndsWith(".launcher", StringComparison.OrdinalIgnoreCase) + || package.EndsWith(".launcher2", StringComparison.OrdinalIgnoreCase) + || package.EndsWith(".launcher3", StringComparison.OrdinalIgnoreCase) + || package.EndsWith(".home", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (!string.IsNullOrWhiteSpace(activity)) + { + var simple = activity.LastIndexOf('.') is >= 0 and var dot ? activity[(dot + 1)..] : activity; + if (simple.EndsWith("Launcher", StringComparison.OrdinalIgnoreCase) + || simple.EndsWith("LauncherActivity", StringComparison.OrdinalIgnoreCase) + || simple.Equals("HomeActivity", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + /// + /// Parses dumpsys window (or dumpsys window displays, or filtered mCurrentFocus/mFocusedApp + /// lines) and resolves the active display and foreground application. + /// + /// Output of dumpsys window. + /// Optional display id reported by emulator host IPC (e.g. MuMu external_renderer_ipc). + public static ActiveDisplayInfo Parse(string? dumpsysWindow, int? preferredDisplayId = null) + { + if (string.IsNullOrWhiteSpace(dumpsysWindow)) + { + return ActiveDisplayInfo.Default; + } + + // Skip any earlier "WINDOW MANAGER LAST ANR" snapshot that precedes live display contents. + var lastHeader = dumpsysWindow.LastIndexOf(DisplayContentsHeader, StringComparison.OrdinalIgnoreCase); + var live = lastHeader >= 0 ? dumpsysWindow.AsSpan(lastHeader) : dumpsysWindow.AsSpan(); + + int? topFocusedDisplayId = null; + var displays = new List(); + MutableDisplay? current = null; + var hasExplicitDisplayHeaders = false; + var inDisplayContents = lastHeader >= 0; + var syntheticDisplayId = 0; + + foreach (var rawLine in live.EnumerateLines()) + { + var line = rawLine.Trim(); + if (line.IsEmpty) + { + continue; + } + + if (line.StartsWith(SectionHeaderPrefix, StringComparison.OrdinalIgnoreCase)) + { + inDisplayContents = line.StartsWith(DisplayContentsHeader, StringComparison.OrdinalIgnoreCase); + if (!inDisplayContents) + { + current = null; + } + + continue; + } + + var lineStr = line.ToString(); + + if (TryParseIntAfter(lineStr, "mTopFocusedDisplayId=", out var topId)) + { + topFocusedDisplayId = topId; + continue; + } + + if (TryParseDisplayHeader(lineStr, out var displayId)) + { + hasExplicitDisplayHeaders = true; + current = new MutableDisplay(displayId); + displays.Add(current); + continue; + } + + if (current is not null && current.Size.IsEmpty && lineStr.Contains("init=", StringComparison.Ordinal)) + { + ParseDisplayGeometry(lineStr, current); + continue; + } + + if (lineStr.StartsWith("mCurrentFocus=", StringComparison.Ordinal)) + { + if (!hasExplicitDisplayHeaders && (!inDisplayContents || lastHeader < 0)) + { + // Headerless / grep-filtered output: pair each mCurrentFocus + mFocusedApp group sequentially. + if (current is null || current.SawCurrentFocus) + { + current = new MutableDisplay(syntheticDisplayId++); + displays.Add(current); + } + } + + if (current is not null) + { + current.SawCurrentFocus = true; + current.CurrentFocus = ParseComponentFromToken(lineStr["mCurrentFocus=".Length..]); + } + + continue; + } + + if (lineStr.StartsWith("mFocusedApp=", StringComparison.Ordinal)) + { + if (!hasExplicitDisplayHeaders && (!inDisplayContents || lastHeader < 0)) + { + if (current is null || current.SawFocusedApp) + { + current = new MutableDisplay(syntheticDisplayId++); + displays.Add(current); + } + } + + if (current is not null) + { + current.SawFocusedApp = true; + current.FocusedApp = ParseComponentFromToken(lineStr["mFocusedApp=".Length..]); + } + } + } + + if (displays.Count == 0) + { + return ActiveDisplayInfo.Default; + } + + // Deduplicate by DisplayId (keep the first occurrence in live display contents, merging missing fields if any). + var entries = new List(displays.Count); + var byId = new Dictionary(); + foreach (var d in displays) + { + var entry = new DisplayFocusEntry(d.DisplayId, d.Size, d.Density, d.CurrentFocus, d.FocusedApp); + if (byId.TryGetValue(d.DisplayId, out var existingIdx)) + { + var prev = entries[existingIdx]; + entries[existingIdx] = new DisplayFocusEntry( + prev.DisplayId, + prev.Size.IsEmpty ? entry.Size : prev.Size, + prev.Density > 0 ? prev.Density : entry.Density, + prev.CurrentFocus ?? entry.CurrentFocus, + prev.FocusedApp ?? entry.FocusedApp); + } + else + { + byId[d.DisplayId] = entries.Count; + entries.Add(entry); + } + } + + var chosen = SelectActiveDisplay(entries, topFocusedDisplayId, preferredDisplayId, hasExplicitDisplayHeaders); + var resolvedId = hasExplicitDisplayHeaders ? chosen.DisplayId : (preferredDisplayId ?? topFocusedDisplayId ?? 0); + return new ActiveDisplayInfo(resolvedId, chosen.Size, chosen.Density, chosen.EffectiveApp, entries); + } + + /// Parses a single Window{…} or ActivityRecord{…} value into an . + public static AppComponent? ParseComponentFromToken(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + if (trimmed.Equals("null", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var m = ComponentRegex().Match(trimmed); + if (!m.Success) + { + return null; + } + + var pkg = m.Groups["pkg"].Value; + var act = m.Groups["act"].Value; + return string.IsNullOrEmpty(pkg) ? null : new AppComponent(pkg, string.IsNullOrEmpty(act) ? null : act); + } + + private static DisplayFocusEntry SelectActiveDisplay( + List entries, + int? topFocusedDisplayId, + int? preferredDisplayId, + bool hasExplicitDisplayHeaders) + { + // 1. Host emulator IPC preference (e.g. MuMu frontmost tab display id), when that display has a running app + // or when it explicitly selects the primary display and no secondary virtual display holds a non-launcher game. + if (preferredDisplayId is int pref + && entries.FirstOrDefault(e => e.DisplayId == pref) is { } prefEntry + && (prefEntry.HasNonLauncherApp || (pref == 0 && !entries.Any(e => e.DisplayId > 0 && e.HasNonLauncherFocus)))) + { + return prefEntry; + } + + // 2. Android's mTopFocusedDisplayId when it points to a display with an active non-launcher window. + if (topFocusedDisplayId is int topId + && entries.FirstOrDefault(e => e.DisplayId == topId) is { } topEntry + && topEntry.HasNonLauncherFocus) + { + return topEntry; + } + + // 3. MuMu 5/6 app_keptlive / multi-tab virtual displays: a secondary display (DisplayId > 0) has a live + // non-launcher mCurrentFocus while Display 0 sits on the launcher (or has mCurrentFocus=null). + var secondaryWithFocus = entries.Where(e => e.DisplayId > 0 && e.HasNonLauncherFocus).ToList(); + if (secondaryWithFocus.Count > 0) + { + // DisplayContents in AOSP dumpsys are emitted top-most/MRU first when organized; in headerless sequential + // output the virtual display appears last. + return hasExplicitDisplayHeaders ? secondaryWithFocus[0] : secondaryWithFocus[^1]; + } + + // 4. Any display (including Display 0) with a non-launcher mCurrentFocus. + var anyWithFocus = entries.Where(e => e.HasNonLauncherFocus).ToList(); + if (anyWithFocus.Count > 0) + { + return hasExplicitDisplayHeaders ? anyWithFocus[0] : anyWithFocus[^1]; + } + + // 5. Secondary or primary display with a non-launcher mFocusedApp (e.g. during splash transition while mCurrentFocus is null). + if (topFocusedDisplayId is int topAppId + && entries.FirstOrDefault(e => e.DisplayId == topAppId) is { HasNonLauncherApp: true } topAppEntry) + { + return topAppEntry; + } + + var secondaryWithApp = entries.Where(e => e.DisplayId > 0 && e.HasNonLauncherApp).ToList(); + if (secondaryWithApp.Count > 0) + { + return hasExplicitDisplayHeaders ? secondaryWithApp[0] : secondaryWithApp[^1]; + } + + var anyWithApp = entries.Where(e => e.HasNonLauncherApp).ToList(); + if (anyWithApp.Count > 0) + { + return hasExplicitDisplayHeaders ? anyWithApp[0] : anyWithApp[^1]; + } + + // 6. Fallback: preferred -> mTopFocusedDisplayId -> Display 0 -> last entry with any app -> first entry. + if (preferredDisplayId is int prefFallback && entries.FirstOrDefault(e => e.DisplayId == prefFallback) is { } pf) + { + return pf; + } + + if (topFocusedDisplayId is int topFallback && entries.FirstOrDefault(e => e.DisplayId == topFallback) is { } tf) + { + return tf; + } + + return entries.FirstOrDefault(e => e.DisplayId == 0 && e.EffectiveApp is not null) + ?? entries.LastOrDefault(e => e.EffectiveApp is not null) + ?? entries.FirstOrDefault(e => e.DisplayId == 0) + ?? entries[0]; + } + + private static bool TryParseDisplayHeader(string line, out int displayId) + { + displayId = 0; + var m = DisplayHeaderRegex().Match(line); + return m.Success && int.TryParse(m.Groups["id"].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out displayId); + } + + private static bool TryParseIntAfter(string line, string prefix, out int value) + { + value = 0; + var idx = line.IndexOf(prefix, StringComparison.Ordinal); + if (idx < 0) + { + return false; + } + + var span = line.AsSpan(idx + prefix.Length).TrimStart(); + var len = 0; + if (len < span.Length && span[len] == '-') + { + len++; + } + + while (len < span.Length && char.IsAsciiDigit(span[len])) + { + len++; + } + + return len > 0 && int.TryParse(span[..len], NumberStyles.Integer, CultureInfo.InvariantCulture, out value); + } + + private static void ParseDisplayGeometry(string line, MutableDisplay target) + { + SizeI initSize = default, curSize = default; + var initMatch = InitGeometryRegex().Match(line); + if (initMatch.Success + && int.TryParse(initMatch.Groups["w"].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out var iw) + && int.TryParse(initMatch.Groups["h"].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out var ih) + && iw > 0 && ih > 0) + { + initSize = new SizeI(iw, ih); + if (initMatch.Groups["dpi"].Success + && int.TryParse(initMatch.Groups["dpi"].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out var dpi) + && dpi > 0) + { + target.Density = dpi; + } + } + + var curMatch = CurGeometryRegex().Match(line); + if (curMatch.Success + && int.TryParse(curMatch.Groups["w"].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out var cw) + && int.TryParse(curMatch.Groups["h"].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out var ch) + && cw > 0 && ch > 0) + { + curSize = new SizeI(cw, ch); + } + + target.Size = !curSize.IsEmpty ? curSize : initSize; + } + + private sealed class MutableDisplay(int displayId) + { + public int DisplayId { get; } = displayId; + + public SizeI Size { get; set; } + + public int Density { get; set; } + + public AppComponent? CurrentFocus { get; set; } + + public AppComponent? FocusedApp { get; set; } + + public bool SawCurrentFocus { get; set; } + + public bool SawFocusedApp { get; set; } + } + + [GeneratedRegex(@"^(?:Display:\s*mDisplayId=|Display\s*#?\s*)(?\d+)\b", RegexOptions.CultureInvariant)] + private static partial Regex DisplayHeaderRegex(); + + [GeneratedRegex(@"\binit=(?\d+)x(?\d+)(?:\s+(?\d+)dpi)?", RegexOptions.CultureInvariant)] + private static partial Regex InitGeometryRegex(); + + [GeneratedRegex(@"\bcur=(?\d+)x(?\d+)", RegexOptions.CultureInvariant)] + private static partial Regex CurGeometryRegex(); + + [GeneratedRegex(@"(?[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)+)/(?\.?[A-Za-z0-9_$.]+)", RegexOptions.CultureInvariant)] + private static partial Regex ComponentRegex(); +} diff --git a/src/ClickZen.Device/Input/AdbInputInjector.cs b/src/ClickZen.Device/Input/AdbInputInjector.cs index 582a64f..52e40e4 100644 --- a/src/ClickZen.Device/Input/AdbInputInjector.cs +++ b/src/ClickZen.Device/Input/AdbInputInjector.cs @@ -15,63 +15,76 @@ namespace ClickZen.Device.Input; /// public sealed class AdbInputInjector : ITouchInjector { + /// API level that added input -d <displayId>. + public const int DisplayIdMinSdk = 29; + /// API level that added input motionevent. public const int MotionEventMinSdk = 30; private readonly Func> _shell; private readonly int _sdk; + private readonly Func? _displayId; /// Runs a shell command on the device (e.g. bound to a serial). - public AdbInputInjector(Func> shell, int sdkLevel) + /// Android API level of the device. + /// Optional active Android display id provider (> 0 routes commands via input -d <id>). + public AdbInputInjector(Func> shell, int sdkLevel, Func? displayId = null) { _shell = shell; _sdk = sdkLevel; + _displayId = displayId; } - public static AdbInputInjector For(AdbService adb, string serial, int sdkLevel, bool asRoot = false) => - new(asRoot ? (cmd, ct) => adb.RootShellAsync(serial, cmd, ct) : (cmd, ct) => adb.ShellAsync(serial, cmd, ct), sdkLevel); + public static AdbInputInjector For(AdbService adb, string serial, int sdkLevel, bool asRoot = false, Func? displayId = null) => + new(asRoot ? (cmd, ct) => adb.RootShellAsync(serial, cmd, ct) : (cmd, ct) => adb.ShellAsync(serial, cmd, ct), sdkLevel, displayId); public string Name => "adb"; public bool SupportsMultiTouch => false; + private int TargetDisplayId => _sdk >= DisplayIdMinSdk && _displayId is not null ? Math.Max(0, _displayId()) : 0; + public Task TapAsync(PointI point, int durationMs, CancellationToken ct) => - _shell(BuildTap(point, durationMs), ct); + _shell(BuildTap(point, durationMs, TargetDisplayId), ct); public Task StrokeAsync(IReadOnlyList path, CancellationToken ct) => - path.Count == 0 ? Task.CompletedTask : _shell(BuildStroke(path, _sdk), ct); + path.Count == 0 ? Task.CompletedTask : _shell(BuildStroke(path, _sdk, TargetDisplayId), ct); public Task MultiStrokeAsync(IReadOnlyList> fingers, CancellationToken ct) => fingers.Count == 0 ? Task.CompletedTask : StrokeAsync(fingers[0], ct); public Task KeyAsync(int keyCode, CancellationToken ct) => - _shell("input keyevent " + keyCode.ToString(CultureInfo.InvariantCulture), ct); + _shell(InputPrefix(TargetDisplayId) + " keyevent " + keyCode.ToString(CultureInfo.InvariantCulture), ct); public Task TextAsync(string text, CancellationToken ct) => - string.IsNullOrEmpty(text) ? Task.CompletedTask : _shell("input text " + EscapeInputText(text), ct); + string.IsNullOrEmpty(text) ? Task.CompletedTask : _shell(InputPrefix(TargetDisplayId) + " text " + EscapeInputText(text), ct); // ------------------------------------------------------------------ command builders (pure, tested) - internal static string BuildTap(PointI p, int durationMs) => + private static string InputPrefix(int displayId) => + displayId > 0 ? string.Create(CultureInfo.InvariantCulture, $"input -d {displayId}") : "input"; + + internal static string BuildTap(PointI p, int durationMs, int displayId = 0) => durationMs <= 100 - ? string.Create(CultureInfo.InvariantCulture, $"input tap {p.X} {p.Y}") - : string.Create(CultureInfo.InvariantCulture, $"input swipe {p.X} {p.Y} {p.X} {p.Y} {durationMs}"); + ? string.Create(CultureInfo.InvariantCulture, $"{InputPrefix(displayId)} tap {p.X} {p.Y}") + : string.Create(CultureInfo.InvariantCulture, $"{InputPrefix(displayId)} swipe {p.X} {p.Y} {p.X} {p.Y} {durationMs}"); - internal static string BuildStroke(IReadOnlyList path, int sdk) + internal static string BuildStroke(IReadOnlyList path, int sdk, int displayId = 0) { + var prefix = InputPrefix(displayId); var first = path[0]; var last = path[^1]; var duration = Math.Max(1, last.OffsetMs - first.OffsetMs); if (sdk < MotionEventMinSdk || path.Count == 1) { return string.Create(CultureInfo.InvariantCulture, - $"input swipe {R(first.X)} {R(first.Y)} {R(last.X)} {R(last.Y)} {duration}"); + $"{prefix} swipe {R(first.X)} {R(first.Y)} {R(last.X)} {R(last.Y)} {duration}"); } // One shell invocation: DOWN, MOVEs at ~16 ms cadence with sleeps, UP. var points = Trajectory.ResampleByTime(path, 16); var sb = new StringBuilder(); - sb.Append(CultureInfo.InvariantCulture, $"input motionevent DOWN {R(first.X)} {R(first.Y)}"); + sb.Append(CultureInfo.InvariantCulture, $"{prefix} motionevent DOWN {R(first.X)} {R(first.Y)}"); var prevT = first.OffsetMs; for (var i = 1; i < points.Count; i++) { @@ -82,11 +95,11 @@ internal static string BuildStroke(IReadOnlyList path, int sdk) sb.Append(CultureInfo.InvariantCulture, $"; sleep {gap / 1000.0:0.###}"); } - sb.Append(CultureInfo.InvariantCulture, $"; input motionevent MOVE {R(p.X)} {R(p.Y)}"); + sb.Append(CultureInfo.InvariantCulture, $"; {prefix} motionevent MOVE {R(p.X)} {R(p.Y)}"); prevT = p.OffsetMs; } - sb.Append(CultureInfo.InvariantCulture, $"; input motionevent UP {R(last.X)} {R(last.Y)}"); + sb.Append(CultureInfo.InvariantCulture, $"; {prefix} motionevent UP {R(last.X)} {R(last.Y)}"); return sb.ToString(); } diff --git a/src/ClickZen.Device/Input/DisplayRotationResolver.cs b/src/ClickZen.Device/Input/DisplayRotationResolver.cs index 2a05782..0103a83 100644 --- a/src/ClickZen.Device/Input/DisplayRotationResolver.cs +++ b/src/ClickZen.Device/Input/DisplayRotationResolver.cs @@ -20,39 +20,40 @@ public sealed class FixedRotationResolver(DisplayRotation rotation) : IDisplayRo } /// -/// Reads the rotation from dumpsys input (viewport / SurfaceOrientation of the default display), falling +/// Reads the rotation from dumpsys input (viewport / SurfaceOrientation of the target display), falling /// back to dumpsys window displays. Unknown output means . /// -public sealed class DumpsysRotationResolver(AdbService adb) : IDisplayRotationResolver +public sealed class DumpsysRotationResolver(AdbService adb, Func? displayId = null) : IDisplayRotationResolver { public const string InputCommand = "dumpsys input | grep -E 'Viewport|SurfaceOrientation'"; public const string WindowCommand = "dumpsys window displays | grep -E 'mCurrentRotation|mRotation='"; public async Task GetRotationAsync(string serial, CancellationToken ct) { - if (DisplayRotationParser.TryParse(await adb.ShellAsync(serial, InputCommand, ct), out var rotation)) + var targetDisplay = displayId is not null ? Math.Max(0, displayId()) : 0; + if (DisplayRotationParser.TryParse(await adb.ShellAsync(serial, InputCommand, ct), out var rotation, targetDisplay)) { return rotation; } - return DisplayRotationParser.TryParse(await adb.ShellAsync(serial, WindowCommand, ct), out rotation) + return DisplayRotationParser.TryParse(await adb.ShellAsync(serial, WindowCommand, ct), out rotation, targetDisplay) ? rotation : DisplayRotation.Rotation0; } } -/// Extracts the default display's rotation from various dumpsys formats. +/// Extracts a display's rotation from various dumpsys formats. public static partial class DisplayRotationParser { /// /// Recognises, in order of preference: /// - /// Viewport INTERNAL: displayId=0, … orientation=1, … (dumpsys input, Android 10+) + /// Viewport …: displayId=<id>, … orientation=1, … (dumpsys input, Android 10+) /// SurfaceOrientation: 1 (dumpsys input, touch mapper) /// mCurrentRotation=ROTATION_90 / mRotation=1 (dumpsys window) /// /// - public static bool TryParse(string? output, out DisplayRotation rotation) + public static bool TryParse(string? output, out DisplayRotation rotation, int displayId = 0) { rotation = DisplayRotation.Rotation0; if (string.IsNullOrEmpty(output)) @@ -60,13 +61,34 @@ public static bool TryParse(string? output, out DisplayRotation rotation) return false; } - // Prefer the viewport of display 0; inactive placeholder viewports have displayId=-1. + var target = Math.Max(0, displayId).ToString(CultureInfo.InvariantCulture); + DisplayRotation? displayZeroFallback = null; + + // Prefer the viewport of target displayId, falling back to display 0; inactive placeholder viewports have displayId=-1. foreach (Match m in ViewportRegex().Matches(output)) { - if (TryRotation(m.Groups["r"].Value, out rotation)) + var id = m.Groups["id"].Value; + if (!TryRotation(m.Groups["r"].Value, out var r)) + { + continue; + } + + if (id == target) { + rotation = r; return true; } + + if (id == "0" && displayZeroFallback is null) + { + displayZeroFallback = r; + } + } + + if (displayZeroFallback is { } zeroRot) + { + rotation = zeroRot; + return true; } foreach (var regex in new[] { SurfaceOrientationRegex(), CurrentRotationRegex(), RotationRegex() }) @@ -106,7 +128,7 @@ private static bool TryRotation(string token, out DisplayRotation rotation) return false; } - [GeneratedRegex(@"Viewport\s+\w+:\s*displayId=0\b[^\n]*?\borientation=(?\d+)")] + [GeneratedRegex(@"Viewport\s+\w+:\s*displayId=(?\d+)\b[^\n]*?\borientation=(?\d+)")] private static partial Regex ViewportRegex(); [GeneratedRegex(@"SurfaceOrientation:\s*(?\d+)")] diff --git a/src/ClickZen.Device/Scrcpy/Protocol/ScrcpyServerOptions.cs b/src/ClickZen.Device/Scrcpy/Protocol/ScrcpyServerOptions.cs index 73ac333..43b7404 100644 --- a/src/ClickZen.Device/Scrcpy/Protocol/ScrcpyServerOptions.cs +++ b/src/ClickZen.Device/Scrcpy/Protocol/ScrcpyServerOptions.cs @@ -15,6 +15,9 @@ public sealed record ScrcpyServerOptions /// true: PC connects to the device (adb forward); false: device connects to PC (adb reverse). public bool TunnelForward { get; init; } + /// Android display id to mirror and inject into (0 = primary display; > 0 = secondary/virtual display, e.g. MuMu game tab). + public int DisplayId { get; init; } + /// Longest edge of the video, 0 = native. public int MaxSize { get; init; } = 1280; @@ -50,6 +53,11 @@ void P(string key, object value) => P("audio", false); P("control", Control); P("tunnel_forward", TunnelForward); + if (DisplayId > 0) + { + P("display_id", DisplayId); + } + if (Video) { P("video_codec", "h264"); diff --git a/src/ClickZen.Device/Scrcpy/ScrcpySession.cs b/src/ClickZen.Device/Scrcpy/ScrcpySession.cs index 45ac8d5..a4007fc 100644 --- a/src/ClickZen.Device/Scrcpy/ScrcpySession.cs +++ b/src/ClickZen.Device/Scrcpy/ScrcpySession.cs @@ -25,6 +25,7 @@ public enum SessionState public sealed record ScrcpySessionOptions { public bool Video { get; init; } = true; + public int DisplayId { get; init; } public int MaxSize { get; init; } = 1280; public int VideoBitRate { get; init; } = 8_000_000; public int MaxFps { get; init; } = 60; @@ -47,7 +48,11 @@ public sealed class ScrcpySession : ILiveFrameSource, IAsyncDisposable private readonly ILogger? _logger; private readonly IClock _clock; private readonly Lock _frameLock = new(); + private readonly Lock _switchLock = new(); private readonly CancellationTokenSource _life = new(); + private CancellationTokenSource? _attemptCts; + private volatile int _displayId; + private volatile bool _switchingDisplay; private Frame? _latest; private long _sequence; private TaskCompletionSource _frameSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -64,6 +69,7 @@ public ScrcpySession(string serial, IScrcpyTransport transport, ScrcpySessionOpt Serial = serial; _transport = transport; _options = options ?? new ScrcpySessionOptions(); + _displayId = Math.Max(0, _options.DisplayId); _logger = logger; _clock = clock ?? SystemClock.Instance; Injector = new ScrcpyTouchInjector(SendAsync, () => _videoSize, () => DeviceSize, _clock); @@ -71,6 +77,9 @@ public ScrcpySession(string serial, IScrcpyTransport transport, ScrcpySessionOpt public string Serial { get; } + /// Currently mirrored Android display id (0 = primary, > 0 = secondary/virtual display). + public int DisplayId => _displayId; + public string DeviceName { get; private set; } = ""; public ScrcpyTouchInjector Injector { get; } @@ -183,6 +192,35 @@ public async Task StartAsync(CancellationToken ct = default) await firstAttempt.Task; } + /// + /// Switches the session in place to another Android display id (e.g. when a multi-display emulator such as MuMu 12 + /// opens or switches a game tab on a virtual display). Consumers keep the same instance. + /// + public void SwitchDisplay(int displayId) + { + var next = Math.Max(0, displayId); + CancellationTokenSource? toCancel; + lock (_switchLock) + { + if (_displayId == next) + { + return; + } + + _displayId = next; + _switchingDisplay = true; + toCancel = _attemptCts; + } + + try + { + toCancel?.Cancel(); + } + catch (ObjectDisposedException) + { + } + } + private async Task RunAsync(TaskCompletionSource firstAttempt, CancellationToken ct) { var backoff = TimeSpan.FromSeconds(1); @@ -190,11 +228,22 @@ private async Task RunAsync(TaskCompletionSource firstAttempt, CancellationToken { State = _state == SessionState.Disconnected ? SessionState.Connecting : SessionState.Reconnecting; ScrcpyStartResult? started = null; + int targetDisplay; + CancellationTokenSource attemptCts; + lock (_switchLock) + { + _switchingDisplay = false; + targetDisplay = _displayId; + attemptCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + _attemptCts = attemptCts; + } + try { var options = new ScrcpyServerOptions { Scid = ScrcpyServerOptions.NewScid(), + DisplayId = targetDisplay, Video = _options.Video, MaxSize = _options.MaxSize, VideoBitRate = _options.VideoBitRate, @@ -202,43 +251,73 @@ private async Task RunAsync(TaskCompletionSource firstAttempt, CancellationToken StayAwake = _options.StayAwake, ShowTouches = _options.ShowTouches, }; - started = await _transport.StartAsync(Serial, options, ct); + started = await _transport.StartAsync(Serial, options, attemptCts.Token); _connection = started.Connection; DeviceName = started.Connection.DeviceName; LastError = null; backoff = TimeSpan.FromSeconds(1); State = started.Connection.HasVideo ? SessionState.Streaming : SessionState.Connected; firstAttempt.TrySetResult(); - _logger?.LogInformation("scrcpy session for {Serial} ({Name}) connected, video={Video}", Serial, DeviceName, started.Connection.HasVideo); + _logger?.LogInformation("scrcpy session for {Serial} ({Name}) connected on display {DisplayId}, video={Video}", + Serial, DeviceName, targetDisplay, started.Connection.HasVideo); - using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct); - var tasks = new List { started.ServerExited, started.Connection.DrainControlAsync(linked.Token) }; + var tasks = new List { started.ServerExited, started.Connection.DrainControlAsync(attemptCts.Token) }; if (started.Connection.HasVideo) { - tasks.Add(Task.Run(() => PumpVideoAsync(started.Connection, linked.Token), CancellationToken.None)); + tasks.Add(Task.Run(() => PumpVideoAsync(started.Connection, attemptCts.Token), CancellationToken.None)); } var finished = await Task.WhenAny(tasks); - await linked.CancelAsync(); - if (!ct.IsCancellationRequested) + await attemptCts.CancelAsync(); + if (!ct.IsCancellationRequested && !_switchingDisplay) { LastError = finished.Exception?.GetBaseException().Message ?? "Connection closed by the device."; - _logger?.LogWarning("scrcpy session for {Serial} ended: {Error}", Serial, LastError); + _logger?.LogWarning("scrcpy session for {Serial} (display {DisplayId}) ended: {Error}", Serial, targetDisplay, LastError); } } catch (OperationCanceledException) when (ct.IsCancellationRequested) { break; } - catch (Exception ex) + catch (OperationCanceledException) when (_switchingDisplay) { - LastError = ex.Message; - _logger?.LogWarning(ex, "scrcpy session for {Serial} failed to start", Serial); firstAttempt.TrySetResult(); } + catch (Exception ex) + { + if (started is null && targetDisplay > 0) + { + // Virtual display may have just closed (e.g. game tab exited); fall back immediately to primary display 0. + _logger?.LogWarning(ex, "scrcpy session for {Serial} could not attach to display {DisplayId}; falling back to display 0", + Serial, targetDisplay); + lock (_switchLock) + { + if (_displayId == targetDisplay) + { + _displayId = 0; + _switchingDisplay = true; + } + } + } + else + { + LastError = ex.Message; + _logger?.LogWarning(ex, "scrcpy session for {Serial} failed to start", Serial); + firstAttempt.TrySetResult(); + } + } finally { _connection = null; + lock (_switchLock) + { + if (ReferenceEquals(_attemptCts, attemptCts)) + { + _attemptCts = null; + } + } + + attemptCts.Dispose(); if (started is not null) { await started.DisposeAsync(); @@ -250,6 +329,12 @@ private async Task RunAsync(TaskCompletionSource firstAttempt, CancellationToken break; } + if (_switchingDisplay) + { + backoff = TimeSpan.FromSeconds(1); + continue; + } + if (!_options.AutoReconnect) { State = SessionState.Faulted; diff --git a/src/ClickZen.Mcp/ClickZenTools.cs b/src/ClickZen.Mcp/ClickZenTools.cs index 1d4e34f..7ab5934 100644 --- a/src/ClickZen.Mcp/ClickZenTools.cs +++ b/src/ClickZen.Mcp/ClickZenTools.cs @@ -320,6 +320,14 @@ public Task ListApps( return shown.Count == 0 ? "No matching packages." : $"{shown.Count} package(s):\n" + string.Join('\n', shown.Select(a => a.System ? a.Package + " (system)" : a.Package)); }, cancellationToken); + [McpServerTool(Name = "get_foreground_app", Title = "Get foreground app", ReadOnly = true, Idempotent = true, Destructive = false, OpenWorld = false)] + [McpCapability(McpCapability.Observe)] + [Description("Returns the currently focused Android package, activity and display id (handles multi-display emulators such as MuMu 12 where games run on a secondary virtual display). Needs adb.")] + public Task GetForegroundApp( + [Description("Device id or name from list_devices; omit for the current device.")] string? device = null, + CancellationToken cancellationToken = default) => + Run(async ct => McpJson.Serialize(await B.GetForegroundAppAsync(device, ct)), cancellationToken); + // ===================================================================================== // Input // ===================================================================================== diff --git a/src/ClickZen.Mcp/IClickZenBackend.cs b/src/ClickZen.Mcp/IClickZenBackend.cs index 332e370..54d5c55 100644 --- a/src/ClickZen.Mcp/IClickZenBackend.cs +++ b/src/ClickZen.Mcp/IClickZenBackend.cs @@ -26,7 +26,19 @@ public sealed record McpDevice( int Density, string? AndroidVersion, string? InputMethod, - string? Error); + string? Error, + string? ForegroundApp = null, + int DisplayId = 0); + +/// Active display and foreground Android application of a device. +public sealed record McpForegroundApp( + string Device, + int DisplayId, + string? Package, + string? Activity, + string? Component, + int ScreenWidth, + int ScreenHeight); /// An encoded screenshot. /// Encoded image bytes. @@ -211,6 +223,9 @@ public interface IClickZenBackend /// Screen size in the current orientation (input coordinate space). Task GetScreenSizeAsync(string? device, CancellationToken ct); + /// Live foreground package/activity and active Android display id (including secondary virtual displays). + Task GetForegroundAppAsync(string? device, CancellationToken ct); + Task TapAsync(string? device, PointI point, int durationMs, CancellationToken ct); Task SwipeAsync(string? device, IReadOnlyList points, int durationMs, CancellationToken ct); diff --git a/src/ClickZen.Platform/Emulators/EmulatorDetector.cs b/src/ClickZen.Platform/Emulators/EmulatorDetector.cs index c558ec8..a9ed2f9 100644 --- a/src/ClickZen.Platform/Emulators/EmulatorDetector.cs +++ b/src/ClickZen.Platform/Emulators/EmulatorDetector.cs @@ -85,9 +85,10 @@ public EmulatorDetection Detect() // ================================================================== MuMu 12 /// - /// MuMu 12: one player per instance (5.x nx_device\<engine>\shell\MuMuNxDevice.exe, 4.x - /// shell\MuMuPlayer.exe) and one MuMuVMMHeadless.exe --comment <vm> per running VM. The VM name - /// ("MuMuPlayer-12.0-1") comes from either command line; its port from vms\<vm>\configs\vm_config.json. + /// MuMu 12 (4.x / 5.x / 6.x): one player per instance (nx_device\<engine>\shell\MuMuNxDevice.exe -v <index> + /// or shell\MuMuPlayer.exe) and one MuMuVMMHeadless.exe --comment <vm> per running VM. Resolves the + /// instance from the VM name or -v/--vmindex index, reads vms\<vm>\configs\vm_config.json, and + /// falls back to MuMuManager.exe info --vmindex all when elevation hides command lines or VMs live elsewhere. /// private List DetectMuMu(List players, List vms) { @@ -95,20 +96,77 @@ private List DetectMuMu(List players, List EmulatorConfigParsers.FindMuMuVmName(CommandLine.Option(CommandLine.Split(v.CommandLine), "--comment") ?? v.CommandLine)) .OfType() + .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); + var managerCache = new Dictionary>(StringComparer.OrdinalIgnoreCase); + IReadOnlyList GetManagerInfo(string? install) + { + if (string.IsNullOrEmpty(install)) + { + return []; + } + + if (!managerCache.TryGetValue(install, out var cached)) + { + cached = QueryMuMuManager(install); + managerCache[install] = cached; + } + + return cached; + } + var claimed = new HashSet(StringComparer.OrdinalIgnoreCase); + var claimedIndexes = new HashSet(); + void Claim(string vmName, int? idx) + { + claimed.Add(vmName); + if ((idx ?? EmulatorConfigParsers.MuMuVmIndex(vmName)) is int i) + { + claimedIndexes.Add(i); + } + } + foreach (var p in players) { - var install = MuMuInstallDir(p.Path); + var install = MuMuInstallDir(p.Path) ?? InstallDirsFromRegistry(EmulatorCatalog.MuMu).FirstOrDefault(); var vm = EmulatorConfigParsers.FindMuMuVmName(p.CommandLine); + var cmdIndex = EmulatorConfigParsers.MuMuCommandLineIndex(p.CommandLine); + MuMuManagerInstanceInfo? mgrMatch = null; + + if (vm is null && cmdIndex is int explicitIdx) + { + vm = vmNames.FirstOrDefault(n => EmulatorConfigParsers.MuMuVmIndex(n) == explicitIdx) + ?? (install is not null ? FirstMuMuVm(install, explicitIdx) : null) + ?? SynthesizeMuMuVmName(p.Path, explicitIdx); + } + if (vm is null) { - // No instance on the command line (the main instance started from the shortcut): take an unclaimed VM. + // No instance on the command line (started from shortcut without -v, or command line hidden by UAC). var candidates = vmNames.Where(n => !claimed.Contains(n)).ToList(); - vm = candidates.Count == 1 ? candidates[0] - : candidates.FirstOrDefault(n => EmulatorConfigParsers.MuMuVmIndex(n) == 0) - ?? (install is not null ? FirstMuMuVm(install, 0) : null); + if (candidates.Count == 1) + { + vm = candidates[0]; + } + else if (candidates.FirstOrDefault(n => EmulatorConfigParsers.MuMuVmIndex(n) == 0) is { } vmZero) + { + vm = vmZero; + } + else if (install is not null) + { + var runningMgr = GetManagerInfo(install).Where(m => m.IsRunning && !claimedIndexes.Contains(m.Index)).ToList(); + mgrMatch = runningMgr.FirstOrDefault(m => m.ProcessId == p.Id) + ?? (runningMgr.Count == 1 ? runningMgr[0] : runningMgr.FirstOrDefault(m => m.Index == 0) ?? runningMgr.FirstOrDefault()); + if (mgrMatch is not null) + { + vm = FirstMuMuVm(install, mgrMatch.Index) ?? SynthesizeMuMuVmName(p.Path, mgrMatch.Index); + } + else + { + vm = FirstUnclaimedMuMuVm(install, claimed, claimedIndexes); + } + } } if (vm is null) @@ -116,24 +174,48 @@ private List DetectMuMu(List players, List !claimed.Contains(n))) { - claimed.Add(vm); - var install = players.Select(p => MuMuInstallDir(p.Path)).FirstOrDefault(d => d is not null) ?? MuMuInstallFromRegistry(vm); - result.Add(MuMuInstance(vm, install, null)); + var idx = EmulatorConfigParsers.MuMuVmIndex(vm); + Claim(vm, idx); + var install = players.Select(p => MuMuInstallDir(p.Path)).FirstOrDefault(d => d is not null) + ?? MuMuInstallFromRegistry(vm) + ?? InstallDirsFromRegistry(EmulatorCatalog.MuMu).FirstOrDefault(); + result.Add(MuMuInstance(vm, idx, install, null, null, GetManagerInfo)); + } + + // Headless VMs whose command lines were hidden by MuMuVMMSVC (SYSTEM): ask MuMuManager when VMs are running. + if (result.Count == 0 && vms.Count > 0) + { + foreach (var install in InstallDirsFromRegistry(EmulatorCatalog.MuMu)) + { + foreach (var m in GetManagerInfo(install).Where(x => x.IsRunning && !claimedIndexes.Contains(x.Index))) + { + var vm = FirstMuMuVm(install, m.Index) ?? $"MuMuPlayer-12.0-{m.Index.ToString(CultureInfo.InvariantCulture)}"; + Claim(vm, m.Index); + result.Add(MuMuInstance(vm, m.Index, install, null, m, GetManagerInfo)); + } + } } return result; } - private EmulatorInstance MuMuInstance(string vm, string? install, ProcessInfo? player) + private EmulatorInstance MuMuInstance( + string vm, + int? index, + string? install, + ProcessInfo? player, + MuMuManagerInstanceInfo? mgrInfo, + Func> getManagerInfo) { - var index = EmulatorConfigParsers.MuMuVmIndex(vm); + index ??= EmulatorConfigParsers.MuMuVmIndex(vm); int? port = null; string? configPath = null; string? name = null; @@ -153,6 +235,23 @@ private EmulatorInstance MuMuInstance(string vm, string? install, ProcessInfo? p } var source = AdbPortSource.Config; + if (port is null && index is int lookupIdx && install is not null) + { + // VMs may have been relocated to another drive in MuMu Settings; ask MuMuManager if present. + mgrInfo ??= getManagerInfo(install).FirstOrDefault(m => m.Index == lookupIdx); + if (mgrInfo?.AdbPort is int mgrPort) + { + port = mgrPort; + source = AdbPortSource.Runtime; + configPath = null; + } + } + + if (string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(mgrInfo?.Name)) + { + name = mgrInfo.Name; + } + if (port is null && index is int i) { port = EmulatorCatalog.MuMu.PortForIndex(i); @@ -169,6 +268,7 @@ private EmulatorInstance MuMuInstance(string vm, string? install, ProcessInfo? p InstallDirectory = install, ProcessId = player?.Id ?? 0, ProcessName = player?.Name ?? "", + WindowHandle = mgrInfo?.MainWindowHandle ?? 0, AdbPort = port, PortSource = port is null ? AdbPortSource.Unknown : source, // MuMu takes the next free port when its port is in use. @@ -185,18 +285,24 @@ private EmulatorInstance MuMuInstance(string vm, string? install, ProcessInfo? p return null; } + string? markerFallback = null; var dir = Path.GetDirectoryName(playerPath); - for (var up = 0; dir is not null && up < 4; up++) + for (var up = 0; dir is not null && up < 5; up++) { - if (_env.DirectoryExists(Path.Combine(dir, "vms")) && IsInstall(EmulatorCatalog.MuMu, dir)) + if (IsInstall(EmulatorCatalog.MuMu, dir)) { - return dir; + if (_env.DirectoryExists(Path.Combine(dir, "vms"))) + { + return dir; + } + + markerFallback ??= dir; } dir = Path.GetDirectoryName(dir); } - return null; + return markerFallback; } private string? MuMuInstallFromRegistry(string vm) => @@ -207,6 +313,79 @@ private EmulatorInstance MuMuInstance(string vm, string? install, ProcessInfo? p .Select(Path.GetFileName) .FirstOrDefault(n => EmulatorConfigParsers.MuMuVmIndex(n) == index); + private string? FirstUnclaimedMuMuVm(string install, HashSet claimed, HashSet claimedIndexes) + { + var dirs = _env.GetDirectories(Path.Combine(install, "vms")) + .Select(Path.GetFileName) + .OfType() + .Select(n => (Name: n, Index: EmulatorConfigParsers.MuMuVmIndex(n))) + .Where(x => x.Index is int idx && !claimed.Contains(x.Name) && !claimedIndexes.Contains(idx)) + .OrderBy(x => x.Index) + .ToList(); + + // Prefer instance 0 when present; when instance 0 was deleted (e.g. only instance 1 exists), use the sole VM. + return dirs.FirstOrDefault(x => x.Index == 0).Name + ?? (dirs.Count == 1 ? dirs[0].Name : null); + } + + private static string SynthesizeMuMuVmName(string? playerPath, int index) + { + var engine = "12.0"; + if (!string.IsNullOrEmpty(playerPath)) + { + // …\nx_device\15.0\shell\MuMuNxDevice.exe → engine "15.0" + var shellDir = Path.GetDirectoryName(playerPath); + var engineDir = shellDir is null ? null : Path.GetDirectoryName(shellDir); + var parentDir = engineDir is null ? null : Path.GetDirectoryName(engineDir); + if (parentDir is not null + && string.Equals(Path.GetFileName(parentDir), "nx_device", StringComparison.OrdinalIgnoreCase) + && Path.GetFileName(engineDir) is { Length: > 0 } ver) + { + engine = ver; + } + } + + return string.Create(CultureInfo.InvariantCulture, $"MuMuPlayer-{engine}-{index}"); + } + + private IReadOnlyList QueryMuMuManager(string install) + { + string? exe = null; + foreach (var rel in new[] { @"nx_main\MuMuManager.exe", @"shell\MuMuManager.exe", @"nx_main\mumu-cli.exe" }) + { + var candidate = Path.Combine(install, rel); + if (_env.FileExists(candidate)) + { + exe = candidate; + break; + } + } + + if (exe is null) + { + return []; + } + + var infoJson = _env.RunCommand(exe, "info --vmindex all"); + if (string.IsNullOrWhiteSpace(infoJson)) + { + return []; + } + + var parsed = EmulatorConfigParsers.ParseMuMuManagerInfo(infoJson); + if (parsed.Any(p => p.IsRunning && p.AdbPort is null)) + { + // MuMu v6 can omit adb_port from `info` and expose it via `adb --vmindex all` (MaaFramework #1395). + var adbJson = _env.RunCommand(exe, "adb --vmindex all"); + if (!string.IsNullOrWhiteSpace(adbJson)) + { + parsed = EmulatorConfigParsers.ParseMuMuManagerInfo(infoJson, adbJson); + } + } + + return parsed; + } + // ================================================================== LDPlayer /// @@ -477,9 +656,27 @@ private List DetectAvd(List players) // ================================================================== windows - /// The instance's main window: a window of its player process, preferring the known main window class. + /// The instance's main window: an explicit handle from CLI metadata when present, else a window of its player process. private static EmulatorInstance AttachWindow(EmulatorInstance instance, EmulatorDefinition def, IReadOnlyList windows) { + if (instance.WindowHandle != 0) + { + var byHandle = windows.FirstOrDefault(w => w.Handle == instance.WindowHandle); + if (byHandle is not null) + { + return instance with + { + ProcessId = instance.ProcessId != 0 ? instance.ProcessId : (int)byHandle.ProcessId, + ProcessName = instance.ProcessName.Length > 0 ? instance.ProcessName : byHandle.ProcessName, + WindowHandle = byHandle.Handle, + WindowTitle = byHandle.Title, + WindowClass = byHandle.ClassName, + WindowMinimized = byHandle.IsMinimized, + DisplayName = instance.DisplayName.Length > 0 ? instance.DisplayName : byHandle.Title, + }; + } + } + if (instance.ProcessId == 0) { return instance; @@ -491,7 +688,10 @@ private static EmulatorInstance AttachWindow(EmulatorInstance instance, Emulator return instance; } - var best = own + var primary = own.Where(w => !WindowEnumerator.IsAuxiliaryEmulatorWindow(w.Title, w.ClassName)).ToList(); + var candidates = primary.Count > 0 ? primary : own; + + var best = candidates .OrderByDescending(w => def.MainWindowClasses.Contains(w.ClassName, StringComparer.Ordinal)) .ThenBy(w => w.IsMinimized) .ThenBy(w => w.Title.Length) @@ -586,16 +786,22 @@ void Add(string? dir) } } - const string uninstall = @"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall"; - foreach (var machine in new[] { true, false }) + foreach (var uninstall in new[] + { + @"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall", + @"SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall", + }) { - foreach (var sub in _env.GetRegistrySubKeys(machine, uninstall) - .Where(k => def.UninstallKeyPrefixes.Any(p => k.StartsWith(p, StringComparison.OrdinalIgnoreCase)))) + foreach (var machine in new[] { true, false }) { - var key = uninstall + "\\" + sub; - Add(_env.ReadRegistryString(machine, key, "InstallLocation")); - Add(ExecutableDirectory(_env.ReadRegistryString(machine, key, "UninstallString"))); - Add(ExecutableDirectory(_env.ReadRegistryString(machine, key, "DisplayIcon"))); + foreach (var sub in _env.GetRegistrySubKeys(machine, uninstall) + .Where(k => def.UninstallKeyPrefixes.Any(p => k.StartsWith(p, StringComparison.OrdinalIgnoreCase)))) + { + var key = uninstall + "\\" + sub; + Add(_env.ReadRegistryString(machine, key, "InstallLocation")); + Add(ExecutableDirectory(_env.ReadRegistryString(machine, key, "UninstallString"))); + Add(ExecutableDirectory(_env.ReadRegistryString(machine, key, "DisplayIcon"))); + } } } diff --git a/src/ClickZen.Platform/Emulators/EmulatorEnvironment.cs b/src/ClickZen.Platform/Emulators/EmulatorEnvironment.cs index 0998cfa..5a31009 100644 --- a/src/ClickZen.Platform/Emulators/EmulatorEnvironment.cs +++ b/src/ClickZen.Platform/Emulators/EmulatorEnvironment.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; +using System.Text; using ClickZen.Platform.Windows; using Microsoft.Win32; @@ -36,6 +38,12 @@ public interface IEmulatorEnvironment /// Expands environment variables such as %TEMP% and %LOCALAPPDATA%. string ExpandEnvironment(string text); + + /// + /// Runs a short-lived local helper CLI (used only as a fallback for MuMuManager.exe info --vmindex all + /// when process command lines are unreadable or VMs live outside the install folder); returns stdout or null. + /// + string? RunCommand(string fileName, string arguments, int timeoutMs = 1500) => null; } /// The real system: processes, files, registry and windows of this PC. @@ -145,4 +153,55 @@ public IReadOnlyList GetRegistrySubKeys(bool localMachine, string subKey } public string ExpandEnvironment(string text) => Environment.ExpandEnvironmentVariables(text); + + public string? RunCommand(string fileName, string arguments, int timeoutMs = 1500) + { + if (string.IsNullOrWhiteSpace(fileName) || !File.Exists(fileName)) + { + return null; + } + + try + { + using var proc = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + WorkingDirectory = Path.GetDirectoryName(fileName) ?? "", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + StandardOutputEncoding = Encoding.UTF8, + }, + }; + if (!proc.Start()) + { + return null; + } + + var stdoutTask = proc.StandardOutput.ReadToEndAsync(); + _ = proc.StandardError.ReadToEndAsync(); + if (!proc.WaitForExit(Math.Clamp(timeoutMs, 100, 10_000))) + { + try + { + proc.Kill(entireProcessTree: true); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + } + + return null; + } + + return stdoutTask.GetAwaiter().GetResult(); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + return null; + } + } } diff --git a/src/ClickZen.Platform/Emulators/MuMuExternalRenderer.cs b/src/ClickZen.Platform/Emulators/MuMuExternalRenderer.cs new file mode 100644 index 0000000..bbcab3d --- /dev/null +++ b/src/ClickZen.Platform/Emulators/MuMuExternalRenderer.cs @@ -0,0 +1,283 @@ +using System.Collections.Concurrent; +using System.Runtime.InteropServices; +using ClickZen.Core.Geometry; + +namespace ClickZen.Platform.Emulators; + +/// +/// Optional bridge to MuMu 12's official external_renderer_ipc.dll SDK (modeled on MaaFramework's +/// MuMuPlayerExtras and MAA's MumuExtras). Resolves the active virtual display id in multi-tab / +/// app_keptlive mode and can capture secondary virtual displays directly when screencap only sees display 0. +/// +public static class MuMuExternalRenderer +{ + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int NemuConnectDelegate([MarshalAs(UnmanagedType.LPWStr)] string folder, int index); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate void NemuDisconnectDelegate(int handle); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int NemuGetDisplayIdDelegate(int handle, [MarshalAs(UnmanagedType.LPUTF8Str)] string pkg, int appIndex); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int NemuCaptureDisplayDelegate(int handle, uint displayId, int bufferSize, ref int width, ref int height, nint pixels); + + private sealed record SdkApi( + NemuConnectDelegate Connect, + NemuDisconnectDelegate Disconnect, + NemuGetDisplayIdDelegate? GetDisplayId, + NemuCaptureDisplayDelegate CaptureDisplay); + + private static readonly ConcurrentDictionary LoadedSdks = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Locates external_renderer_ipc.dll inside a MuMu installation root, checking MuMu 6.x (nx_device\15.0), + /// MuMu 5.x (nx_device\12.0 or newer engines), nx_main\sdk, and MuMu 4.x (shell\sdk). + /// + public static string? FindSdkDll( + string? installDir, + Func>? getDirectories = null, + Func? fileExists = null) + { + if (string.IsNullOrWhiteSpace(installDir)) + { + return null; + } + + var exists = fileExists ?? File.Exists; + var listDirs = getDirectories ?? SafeGetDirectories; + var root = installDir.Trim().TrimEnd('\\', '/'); + + // Explicit engine order matching MAA (15.0 -> 12.0), plus any future nx_device\\shell\sdk\ directory. + var candidates = new List + { + Path.Combine(root, "nx_device", "15.0", "shell", "sdk", "external_renderer_ipc.dll"), + Path.Combine(root, "nx_device", "12.0", "shell", "sdk", "external_renderer_ipc.dll"), + }; + + var nxDevice = Path.Combine(root, "nx_device"); + foreach (var dir in listDirs(nxDevice).OrderByDescending(Path.GetFileName, StringComparer.OrdinalIgnoreCase)) + { + var dynamicDll = Path.Combine(dir, "shell", "sdk", "external_renderer_ipc.dll"); + if (!candidates.Contains(dynamicDll, StringComparer.OrdinalIgnoreCase)) + { + candidates.Add(dynamicDll); + } + } + + candidates.Add(Path.Combine(root, "nx_main", "sdk", "external_renderer_ipc.dll")); + candidates.Add(Path.Combine(root, "shell", "sdk", "external_renderer_ipc.dll")); + + return candidates.FirstOrDefault(exists); + } + + /// + /// Queries MuMu's host IPC for the active virtual display id of instance . + /// Checks first when given, then MuMu's "default" sentinel (the + /// currently selected tab in MuMu's UI). Returns null if the SDK DLL is unavailable or the call fails. + /// + public static int? TryGetActiveDisplayId(string? installDir, int instanceIndex, string? foregroundPackage = null) + { + if (string.IsNullOrWhiteSpace(installDir) || instanceIndex < 0) + { + return null; + } + + var api = LoadApi(installDir); + if (api?.GetDisplayId is null) + { + return null; + } + + var root = installDir.Trim().TrimEnd('\\', '/'); + var handle = 0; + try + { + handle = api.Connect(root, instanceIndex); + if (handle <= 0) + { + return null; + } + + // "default" returns the frontmost tab currently selected in MuMu's UI (per MAA MumuExtras.cpp). + var front = api.GetDisplayId(handle, "default", 0); + if (front > 0) + { + return front; + } + + if (!string.IsNullOrWhiteSpace(foregroundPackage)) + { + var pkgId = api.GetDisplayId(handle, foregroundPackage, 0); + if (pkgId >= 0) + { + return pkgId; + } + } + + return front == 0 ? 0 : null; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + return null; + } + finally + { + if (handle > 0) + { + try + { + api.Disconnect(handle); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + } + } + } + } + + /// + /// Captures of a local MuMu instance directly as top-down BGRA pixels via + /// external_renderer_ipc.dll, or returns null if unavailable. + /// + public static (SizeI Size, byte[] Bgra)? TryCaptureDisplayBgra(string? installDir, int instanceIndex, int displayId) + { + if (string.IsNullOrWhiteSpace(installDir) || instanceIndex < 0 || displayId < 0) + { + return null; + } + + var api = LoadApi(installDir); + if (api is null) + { + return null; + } + + var root = installDir.Trim().TrimEnd('\\', '/'); + var handle = 0; + try + { + handle = api.Connect(root, instanceIndex); + if (handle <= 0) + { + return null; + } + + int width = 0, height = 0; + if (api.CaptureDisplay(handle, (uint)displayId, 0, ref width, ref height, 0) != 0 || width <= 0 || height <= 0 || width > 8192 || height > 8192) + { + return null; + } + + var byteCount = width * height * 4; + var raw = new byte[byteCount]; + var pinned = GCHandle.Alloc(raw, GCHandleType.Pinned); + int rc; + try + { + rc = api.CaptureDisplay(handle, (uint)displayId, byteCount, ref width, ref height, pinned.AddrOfPinnedObject()); + } + finally + { + pinned.Free(); + } + + if (rc != 0) + { + return null; + } + + return (new SizeI(width, height), ConvertBottomUpRgbaToTopDownBgra(raw, width, height)); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + return null; + } + finally + { + if (handle > 0) + { + try + { + api.Disconnect(handle); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + } + } + } + } + + /// + /// MuMu's nemu_capture_display writes OpenGL-style bottom-up RGBA; ClickZen frames are top-down BGRA. + /// Pure helper (tested directly). + /// + internal static byte[] ConvertBottomUpRgbaToTopDownBgra(byte[] rgbaBottomUp, int width, int height) + { + var stride = width * 4; + var bgra = new byte[stride * height]; + for (var y = 0; y < height; y++) + { + var srcRow = (height - 1 - y) * stride; + var dstRow = y * stride; + for (var x = 0; x < stride; x += 4) + { + bgra[dstRow + x] = rgbaBottomUp[srcRow + x + 2]; // B + bgra[dstRow + x + 1] = rgbaBottomUp[srcRow + x + 1]; // G + bgra[dstRow + x + 2] = rgbaBottomUp[srcRow + x]; // R + bgra[dstRow + x + 3] = rgbaBottomUp[srcRow + x + 3]; // A + } + } + + return bgra; + } + + private static SdkApi? LoadApi(string installDir) + { + var dll = FindSdkDll(installDir); + return dll is null ? null : LoadedSdks.GetOrAdd(dll, static path => + { + try + { + if (!NativeLibrary.TryLoad(path, out var lib)) + { + return null; + } + + if (!NativeLibrary.TryGetExport(lib, "nemu_connect", out var pConnect) + || !NativeLibrary.TryGetExport(lib, "nemu_disconnect", out var pDisconnect) + || !NativeLibrary.TryGetExport(lib, "nemu_capture_display", out var pCapture)) + { + return null; + } + + var getDisplayId = NativeLibrary.TryGetExport(lib, "nemu_get_display_id", out var pGetDisplay) + ? Marshal.GetDelegateForFunctionPointer(pGetDisplay) + : null; + + return new SdkApi( + Marshal.GetDelegateForFunctionPointer(pConnect), + Marshal.GetDelegateForFunctionPointer(pDisconnect), + getDisplayId, + Marshal.GetDelegateForFunctionPointer(pCapture)); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + return null; + } + }); + } + + private static IReadOnlyList SafeGetDirectories(string path) + { + try + { + return Directory.Exists(path) ? Directory.GetDirectories(path) : []; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + return []; + } + } +} diff --git a/src/ClickZen.Platform/Windows/WindowEnumerator.cs b/src/ClickZen.Platform/Windows/WindowEnumerator.cs index dd13aba..e6109a0 100644 --- a/src/ClickZen.Platform/Windows/WindowEnumerator.cs +++ b/src/ClickZen.Platform/Windows/WindowEnumerator.cs @@ -1,4 +1,5 @@ using ClickZen.Core.Devices; +using ClickZen.Core.Emulators; using ClickZen.Platform.Native; using Windows.Win32; using Windows.Win32.Foundation; @@ -18,8 +19,19 @@ public static class WindowEnumerator private static readonly HashSet IgnoredClasses = new(StringComparer.Ordinal) { "Progman", "Shell_TrayWnd", "Shell_SecondaryTrayWnd", "WorkerW", "Windows.UI.Core.CoreWindow", + "IME", "Default IME", "MSCTFIME UI", "NVOpenGLPbuffer", }; + private static readonly HashSet AuxiliaryTitles = new(StringComparer.OrdinalIgnoreCase) + { + "HintWnd", "MessageWnd", "__wglDummyWindowFodder", + }; + + /// True when a window title/class is an invisible/helper popup owned by an emulator process (e.g. MuMu Qt helpers). + public static bool IsAuxiliaryEmulatorWindow(string? title, string? className) => + (!string.IsNullOrEmpty(title) && AuxiliaryTitles.Contains(title.Trim())) + || (!string.IsNullOrEmpty(className) && (IgnoredClasses.Contains(className) || className.EndsWith("ToolSaveBits", StringComparison.Ordinal))); + /// Visible, non-minimized top-level windows with a title (what can be captured right now). public static IReadOnlyList GetVisibleWindows(uint excludeProcessId = 0) => Enumerate(excludeProcessId, includeMinimized: false); @@ -34,7 +46,7 @@ private static List Enumerate(uint excludeProcessId, bool inc bool Visit(HWND hwnd) { - if (hwnd == shell || !PInvoke.IsWindowVisible(hwnd)) + if (hwnd == shell || !PInvoke.IsWindowVisible(hwnd) || IsCloaked(hwnd)) { return true; } @@ -45,34 +57,47 @@ bool Visit(HWND hwnd) return true; } - var exStyle = (WINDOW_EX_STYLE)PInvoke.GetWindowLong(hwnd, WINDOW_LONG_PTR_INDEX.GWL_EXSTYLE); - if ((exStyle & WINDOW_EX_STYLE.WS_EX_TOOLWINDOW) != 0 || IsCloaked(hwnd)) + var cls = GetClassName(hwnd); + if (IgnoredClasses.Contains(cls) || cls.EndsWith("ToolSaveBits", StringComparison.Ordinal)) { return true; } - var title = GetTitle(hwnd); - if (string.IsNullOrWhiteSpace(title)) + _ = PInvoke.GetWindowThreadProcessId(hwnd, out var pid); + if (excludeProcessId != 0 && pid == excludeProcessId) { return true; } - var cls = GetClassName(hwnd); - if (IgnoredClasses.Contains(cls)) + if (!names.TryGetValue(pid, out var processName)) + { + processName = GetProcessName(pid); + names[pid] = processName; + } + + var isEmulatorPlayer = !string.IsNullOrEmpty(processName) && EmulatorCatalog.All.Any(d => d.IsPlayerProcess(processName)); + + var exStyle = (WINDOW_EX_STYLE)PInvoke.GetWindowLong(hwnd, WINDOW_LONG_PTR_INDEX.GWL_EXSTYLE); + if ((exStyle & WINDOW_EX_STYLE.WS_EX_TOOLWINDOW) != 0 + && !(isEmulatorPlayer && (minimized || HasSubstantialClientArea(hwnd)))) { return true; } - _ = PInvoke.GetWindowThreadProcessId(hwnd, out var pid); - if (excludeProcessId != 0 && pid == excludeProcessId) + var title = GetTitle(hwnd); + if (IsAuxiliaryEmulatorWindow(title, cls)) { return true; } - if (!names.TryGetValue(pid, out var processName)) + if (string.IsNullOrWhiteSpace(title)) { - processName = GetProcessName(pid); - names[pid] = processName; + if (!isEmulatorPlayer || (!minimized && !HasSubstantialClientArea(hwnd))) + { + return true; + } + + title = processName; } result.Add(new DesktopWindowInfo(hwnd, title, cls, pid, processName, minimized)); @@ -130,6 +155,9 @@ public static unsafe string GetProcessName(uint processId) } } + private static bool HasSubstantialClientArea(HWND hwnd) => + PInvoke.GetClientRect(hwnd, out var rc) && (rc.right - rc.left) >= 200 && (rc.bottom - rc.top) >= 200; + private static unsafe bool IsCloaked(HWND hwnd) { int cloaked = 0; @@ -139,17 +167,29 @@ private static unsafe bool IsCloaked(HWND hwnd) private static unsafe string GetTitle(HWND hwnd) { - var len = PInvoke.GetWindowTextLength(hwnd); - if (len <= 0) + char* fast = stackalloc char[512]; + var n = PInvoke.GetWindowText(hwnd, fast, 512); + if (n <= 0) { return ""; } + if (n < 511) + { + return new string(fast, 0, n); + } + + var len = PInvoke.GetWindowTextLength(hwnd); + if (len <= n) + { + return new string(fast, 0, n); + } + var buffer = new char[len + 1]; fixed (char* p = buffer) { - var n = PInvoke.GetWindowText(hwnd, p, buffer.Length); - return new string(p, 0, n); + var full = PInvoke.GetWindowText(hwnd, p, buffer.Length); + return full > 0 ? new string(p, 0, full) : new string(fast, 0, n); } } diff --git a/tests/ClickZen.Core.Tests/Emulators/EmulatorCatalogTests.cs b/tests/ClickZen.Core.Tests/Emulators/EmulatorCatalogTests.cs index e7b29db..7aec81b 100644 --- a/tests/ClickZen.Core.Tests/Emulators/EmulatorCatalogTests.cs +++ b/tests/ClickZen.Core.Tests/Emulators/EmulatorCatalogTests.cs @@ -11,6 +11,8 @@ public sealed class EmulatorCatalogTests [InlineData(EmulatorKind.MuMu, 0, 16384)] [InlineData(EmulatorKind.MuMu, 1, 16416)] [InlineData(EmulatorKind.MuMu, 3, 16480)] + [InlineData(EmulatorKind.MuMu, 32, 16388)] + [InlineData(EmulatorKind.MuMu, 33, 16420)] [InlineData(EmulatorKind.LDPlayer, 0, 5555)] [InlineData(EmulatorKind.LDPlayer, 2, 5559)] [InlineData(EmulatorKind.Nox, 0, 62001)] diff --git a/tests/ClickZen.Core.Tests/Emulators/EmulatorConfigParserTests.cs b/tests/ClickZen.Core.Tests/Emulators/EmulatorConfigParserTests.cs index fb90176..fba5a57 100644 --- a/tests/ClickZen.Core.Tests/Emulators/EmulatorConfigParserTests.cs +++ b/tests/ClickZen.Core.Tests/Emulators/EmulatorConfigParserTests.cs @@ -74,6 +74,75 @@ public void MuMu_vm_index(string? name, int? expected) Assert.Equal(expected is not null, EmulatorConfigParsers.IsMuMuVmName(name)); } + [Theory] + [InlineData("\"D:\\Program Files\\Netease\\MuMu\\nx_device\\12.0\\shell\\MuMuNxDevice.exe\" -v 1", 1)] + [InlineData("\"D:\\MuMu\\nx_device\\15.0\\shell\\MuMuNxDevice.exe\" --vmindex 2", 2)] + [InlineData("MuMuPlayer.exe --index=3", 3)] + [InlineData("MuMuNxDevice.exe -v MuMuPlayer-15.0-4", 4)] + [InlineData("\"D:\\MuMu\\nx_device\\12.0\\shell\\MuMuNxDevice.exe\"", null)] + [InlineData(null, null)] + public void MuMu_command_line_index(string? commandLine, int? expected) + { + Assert.Equal(expected, EmulatorConfigParsers.MuMuCommandLineIndex(commandLine)); + } + + [Theory] + [InlineData(16384, 0)] + [InlineData(16385, 0)] + [InlineData(16416, 1)] + [InlineData(16417, 1)] + [InlineData(16480, 3)] + [InlineData(16388, 32)] + [InlineData(7555, 0)] + [InlineData(5555, null)] + [InlineData(20000, null)] + public void MuMu_port_to_index(int port, int? expected) + { + Assert.Equal(expected, EmulatorConfigParsers.MuMuPortToIndex(port)); + } + + [Fact] + public void MuMu_manager_info_parses_multi_vm_and_v6_adb_fallback() + { + const string info = """ + [INFO] MuMuManager starting... + { + "0": { + "index": "0", + "name": "MuMu模拟器12", + "is_process_started": false, + "is_android_started": false, + "main_wnd": "0", + "render_wnd": "0" + }, + "1": { + "index": "1", + "name": "塔塔冒险队专服", + "is_process_started": true, + "is_android_started": true, + "main_wnd": "1a052c", + "render_wnd": "2b061e" + } + } + """; + const string adb = """ + { + "0": { "adb_host": "127.0.0.1", "adb_port": 16384 }, + "1": { "adb_host": "127.0.0.1", "adb_port": 16416 } + } + """; + + var list = EmulatorConfigParsers.ParseMuMuManagerInfo(info, adb); + Assert.Equal(2, list.Count); + Assert.False(list[0].IsRunning); + Assert.True(list[1].IsRunning); + Assert.Equal(1, list[1].Index); + Assert.Equal("塔塔冒险队专服", list[1].Name); + Assert.Equal(16416, list[1].AdbPort); + Assert.Equal((nint)0x1a052c, list[1].MainWindowHandle); + Assert.Equal((nint)0x2b061e, list[1].RenderWindowHandle); + } + [Fact] public void VBox_forwarding_rule_for_adb() { diff --git a/tests/ClickZen.Device.Tests/Adb/DisplayFocusParserTests.cs b/tests/ClickZen.Device.Tests/Adb/DisplayFocusParserTests.cs new file mode 100644 index 0000000..82d79ee --- /dev/null +++ b/tests/ClickZen.Device.Tests/Adb/DisplayFocusParserTests.cs @@ -0,0 +1,147 @@ +using AdvancedSharpAdbClient.Models; +using ClickZen.Core.Geometry; +using ClickZen.Device.Adb; +using ClickZen.Device.Adb.Parsing; + +namespace ClickZen.Device.Tests.Adb; + +public sealed class DisplayFocusParserTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// Exact verbatim output reported in GitHub Issue #4 (MuMu 12 v6.4.6.0, 127.0.0.1:16416). + private const string Issue4FilteredOutput = """ + mCurrentFocus=null + mFocusedApp=null + mCurrentFocus=null + mFocusedApp=ActivityRecord{24e8c12 u0 app.lawnchair/.LawnchairLauncher t2} + mCurrentFocus=Window{404da34 u0 com.lilithgame.pgame.android.cn/com.farlightgames.igame.SplashActivity} + mFocusedApp=ActivityRecord{1f470d5 u0 com.lilithgame.pgame.android.cn/com.farlightgames.igame.SplashActivity t518} + """; + + private const string FullMuMuMultiDisplayDumpsys = """ + WINDOW MANAGER LAST ANR (dumpsys window lastanr) + Display: mDisplayId=0 + init=1600x900 240dpi cur=1600x900 + mCurrentFocus=Window{deadbeef u0 com.stale.crashed/.CrashActivity} + mFocusedApp=ActivityRecord{deadbeef u0 com.stale.crashed/.CrashActivity t11} + + WINDOW MANAGER DISPLAY CONTENTS (dumpsys window displays) + Display: mDisplayId=1 (organized) + init=1600x900 240dpi cur=1600x900 app=1600x900 + mCurrentFocus=null + mFocusedApp=null + Display: mDisplayId=0 (organized) + init=1600x900 240dpi cur=1600x900 app=1600x900 + mCurrentFocus=null + mFocusedApp=ActivityRecord{24e8c12 u0 app.lawnchair/.LawnchairLauncher t2} + Display: mDisplayId=2 (organized) + init=900x1600 320dpi cur=761x1376 app=761x1376 + mCurrentFocus=Window{404da34 u0 com.lilithgame.pgame.android.cn/com.farlightgames.igame.SplashActivity} + mFocusedApp=ActivityRecord{1f470d5 u0 com.lilithgame.pgame.android.cn/com.farlightgames.igame.SplashActivity t518} + + WINDOW MANAGER WINDOWS (dumpsys window windows) + mGlobalConfiguration={1.0 ?mcc?mnc} + mTopFocusedDisplayId=2 + """; + + [Fact] + public void Issue4_filtered_dumpsys_resolves_foreground_game_over_launcher() + { + var active = DisplayFocusParser.Parse(Issue4FilteredOutput); + + Assert.Equal("com.lilithgame.pgame.android.cn", active.ForegroundPackage); + Assert.Equal("com.farlightgames.igame.SplashActivity", active.ForegroundActivity); + Assert.Equal("com.lilithgame.pgame.android.cn/com.farlightgames.igame.SplashActivity", active.ForegroundComponent); + Assert.Equal(3, active.Displays.Count); + } + + [Fact] + public void Full_mumu_multidisplay_dumpsys_skips_last_anr_and_selects_virtual_display() + { + var active = DisplayFocusParser.Parse(FullMuMuMultiDisplayDumpsys); + + Assert.Equal(2, active.DisplayId); + Assert.Equal(new SizeI(761, 1376), active.Size); + Assert.Equal(320, active.Density); + Assert.Equal("com.lilithgame.pgame.android.cn", active.ForegroundPackage); + Assert.Equal("com.farlightgames.igame.SplashActivity", active.ForegroundActivity); + } + + [Fact] + public void Virtual_display_with_game_wins_even_when_top_focus_briefly_points_to_display0_launcher() + { + var unfocusedHost = FullMuMuMultiDisplayDumpsys.Replace("mTopFocusedDisplayId=2", "mTopFocusedDisplayId=0", StringComparison.Ordinal); + + var active = DisplayFocusParser.Parse(unfocusedHost); + + Assert.Equal(2, active.DisplayId); + Assert.Equal("com.lilithgame.pgame.android.cn", active.ForegroundPackage); + } + + [Fact] + public void Preferred_display_id_from_mumu_ipc_selects_active_tab_between_two_kept_alive_games() + { + const string twoGames = """ + WINDOW MANAGER DISPLAY CONTENTS (dumpsys window displays) + Display: mDisplayId=0 (organized) + init=1600x900 240dpi cur=1600x900 + mCurrentFocus=null + mFocusedApp=ActivityRecord{111 u0 app.lawnchair/.LawnchairLauncher t2} + Display: mDisplayId=2 (organized) + init=1920x1080 320dpi cur=1920x1080 + mCurrentFocus=Window{222 u0 com.hypergryph.arknights/com.u8.sdk.U8UnityContext} + mFocusedApp=ActivityRecord{223 u0 com.hypergryph.arknights/com.u8.sdk.U8UnityContext t10} + Display: mDisplayId=4 (organized) + init=900x1600 320dpi cur=900x1600 + mCurrentFocus=Window{404da34 u0 com.lilithgame.pgame.android.cn/com.farlightgames.igame.SplashActivity} + mFocusedApp=ActivityRecord{1f470d5 u0 com.lilithgame.pgame.android.cn/com.farlightgames.igame.SplashActivity t518} + WINDOW MANAGER WINDOWS (dumpsys window windows) + mTopFocusedDisplayId=2 + """; + + var tab4 = DisplayFocusParser.Parse(twoGames, preferredDisplayId: 4); + Assert.Equal(4, tab4.DisplayId); + Assert.Equal("com.lilithgame.pgame.android.cn", tab4.ForegroundPackage); + Assert.Equal(new SizeI(900, 1600), tab4.Size); + + var tab2 = DisplayFocusParser.Parse(twoGames, preferredDisplayId: 2); + Assert.Equal(2, tab2.DisplayId); + Assert.Equal("com.hypergryph.arknights", tab2.ForegroundPackage); + } + + [Fact] + public void System_window_in_current_focus_falls_back_to_focused_app_on_same_display() + { + const string notificationShade = """ + WINDOW MANAGER DISPLAY CONTENTS (dumpsys window displays) + Display: mDisplayId=0 + init=1080x2400 420dpi cur=1080x2400 + mCurrentFocus=Window{9f812 u0 NotificationShade} + mFocusedApp=ActivityRecord{abc12 u0 com.android.settings/.Settings t9} + """; + + var active = DisplayFocusParser.Parse(notificationShade); + Assert.Equal(0, active.DisplayId); + Assert.Equal("com.android.settings", active.ForegroundPackage); + Assert.Equal(".Settings", active.ForegroundActivity); + } + + [Fact] + public async Task AdbService_GetActiveDisplayAsync_parses_dumpsys_window() + { + var (client, fake) = FakeAdbClient.Create(); + using var svc = new AdbService(client, null, null, new ManualTimeProvider()); + fake.Devices.Add(new DeviceData { Serial = "127.0.0.1:16416", State = DeviceState.Online }); + fake.Shell = (_, cmd, _) => Task.FromResult>( + cmd == "dumpsys window" + ? FullMuMuMultiDisplayDumpsys.Replace("\r", "", StringComparison.Ordinal).Split('\n') + : []); + + var active = await svc.GetActiveDisplayAsync("127.0.0.1:16416", preferredDisplayId: null, Ct); + + Assert.Equal(2, active.DisplayId); + Assert.Equal("com.lilithgame.pgame.android.cn", active.ForegroundPackage); + Assert.Equal(new SizeI(761, 1376), active.Size); + } +} diff --git a/tests/ClickZen.Device.Tests/Input/InputInjectorTests.cs b/tests/ClickZen.Device.Tests/Input/InputInjectorTests.cs index 7494fe5..565b483 100644 --- a/tests/ClickZen.Device.Tests/Input/InputInjectorTests.cs +++ b/tests/ClickZen.Device.Tests/Input/InputInjectorTests.cs @@ -65,6 +65,46 @@ public void Short_tap_uses_input_tap_long_press_uses_swipe() { Assert.Equal("input tap 10 20", AdbInputInjector.BuildTap(new PointI(10, 20), 60)); Assert.Equal("input swipe 10 20 10 20 800", AdbInputInjector.BuildTap(new PointI(10, 20), 800)); + Assert.Equal("input -d 2 tap 10 20", AdbInputInjector.BuildTap(new PointI(10, 20), 60, displayId: 2)); + Assert.Equal("input -d 2 swipe 10 20 10 20 800", AdbInputInjector.BuildTap(new PointI(10, 20), 800, displayId: 2)); + } + + [Fact] + public async Task Secondary_display_injector_passes_dash_d_flag_to_all_commands() + { + var commands = new List(); + var activeDisplay = 2; + var injector = new AdbInputInjector((cmd, _) => + { + commands.Add(cmd); + return Task.FromResult(""); + }, sdkLevel: 32, () => activeDisplay); + + await injector.TapAsync(new PointI(50, 60), 40, TestContext.Current.CancellationToken); + await injector.KeyAsync(4, TestContext.Current.CancellationToken); + await injector.TextAsync("hi", TestContext.Current.CancellationToken); + await injector.StrokeAsync([P(10, 20, 0), P(10, 120, 40)], TestContext.Current.CancellationToken); + + Assert.Equal("input -d 2 tap 50 60", commands[0]); + Assert.Equal("input -d 2 keyevent 4", commands[1]); + Assert.Equal("input -d 2 text hi", commands[2]); + Assert.StartsWith("input -d 2 motionevent DOWN 10 20", commands[3], StringComparison.Ordinal); + Assert.EndsWith("input -d 2 motionevent UP 10 120", commands[3], StringComparison.Ordinal); + } + + [Fact] + public void DisplayRotationParser_prefers_requested_display_id_over_display0() + { + const string viewports = """ + Viewport INTERNAL: displayId=0, uniqueId=local:0, port=0, orientation=1 + Viewport VIRTUAL: displayId=2, uniqueId=virtual:mumu, port=2, orientation=0 + """; + + Assert.True(DisplayRotationParser.TryParse(viewports, out var rot0, displayId: 0)); + Assert.Equal(DisplayRotation.Rotation90, rot0); + + Assert.True(DisplayRotationParser.TryParse(viewports, out var rot2, displayId: 2)); + Assert.Equal(DisplayRotation.Rotation0, rot2); } [Fact] diff --git a/tests/ClickZen.Device.Tests/Scrcpy/ProtocolTests.cs b/tests/ClickZen.Device.Tests/Scrcpy/ProtocolTests.cs index 5f782a6..597af9c 100644 --- a/tests/ClickZen.Device.Tests/Scrcpy/ProtocolTests.cs +++ b/tests/ClickZen.Device.Tests/Scrcpy/ProtocolTests.cs @@ -204,6 +204,14 @@ public void Control_only_omits_video_options() var cmd = new ScrcpyServerOptions { Scid = 1, Video = false }.BuildCommand(); Assert.Contains(" video=false", cmd, StringComparison.Ordinal); Assert.DoesNotContain("max_size", cmd, StringComparison.Ordinal); + Assert.DoesNotContain("display_id", cmd, StringComparison.Ordinal); Assert.Contains(" scid=00000001", cmd, StringComparison.Ordinal); } + + [Fact] + public void Secondary_display_id_is_passed_to_scrcpy_server() + { + var cmd = new ScrcpyServerOptions { Scid = 2, DisplayId = 2 }.BuildCommand(); + Assert.Contains(" display_id=2", cmd, StringComparison.Ordinal); + } } diff --git a/tests/ClickZen.Device.Tests/Scrcpy/ScrcpySessionTests.cs b/tests/ClickZen.Device.Tests/Scrcpy/ScrcpySessionTests.cs index 0f11310..0d19dd6 100644 --- a/tests/ClickZen.Device.Tests/Scrcpy/ScrcpySessionTests.cs +++ b/tests/ClickZen.Device.Tests/Scrcpy/ScrcpySessionTests.cs @@ -218,4 +218,25 @@ public void Device_size_follows_video_orientation() var session = new ScrcpySession("s", new FakeTransport([])) { DeviceSizeOverride = new SizeI(1080, 2400) }; Assert.Equal(new SizeI(1080, 2400), session.DeviceSize); } + + [Fact] + public async Task SwitchDisplay_reconnects_in_place_on_new_display_id() + { + var transport = new FakeTransport([]); + await using var session = new ScrcpySession("127.0.0.1:16416", transport, new ScrcpySessionOptions { AutoReconnect = false }); + + await session.StartAsync(Ct); + Assert.Equal(0, session.DisplayId); + Assert.Equal(0, transport.Options[0].DisplayId); + + session.SwitchDisplay(2); + for (var i = 0; i < 60 && transport.Options.Count < 2; i++) + { + await Task.Delay(25, Ct); + } + + Assert.Equal(2, session.DisplayId); + Assert.True(transport.Options.Count >= 2); + Assert.Equal(2, transport.Options[^1].DisplayId); + } } diff --git a/tests/ClickZen.Mcp.Tests/FakeBackend.cs b/tests/ClickZen.Mcp.Tests/FakeBackend.cs index 95f10af..783dcef 100644 --- a/tests/ClickZen.Mcp.Tests/FakeBackend.cs +++ b/tests/ClickZen.Mcp.Tests/FakeBackend.cs @@ -100,6 +100,9 @@ public Task FindImageAsync(string? device, McpImageQuery query, C public Task GetScreenSizeAsync(string? device, CancellationToken ct) => Task.FromResult(Screen); + public Task GetForegroundAppAsync(string? device, CancellationToken ct) => + Result($"foreground {device}", new McpForegroundApp(device ?? "emulator-5554", 0, "com.demo.game", ".MainActivity", "com.demo.game/.MainActivity", Screen.Width, Screen.Height)); + public Task TapAsync(string? device, PointI point, int durationMs, CancellationToken ct) => Done($"tap {point.X},{point.Y} {durationMs}"); public Task SwipeAsync(string? device, IReadOnlyList points, int durationMs, CancellationToken ct) => diff --git a/tests/ClickZen.Mcp.Tests/McpServerHostTests.cs b/tests/ClickZen.Mcp.Tests/McpServerHostTests.cs index 9aaaef9..d38487f 100644 --- a/tests/ClickZen.Mcp.Tests/McpServerHostTests.cs +++ b/tests/ClickZen.Mcp.Tests/McpServerHostTests.cs @@ -465,4 +465,18 @@ public async Task Bind_emulator_window_returns_the_window_device() Assert.Contains("\"method\":\"screenshotMatch\"", text, StringComparison.Ordinal); Assert.Contains("bind-emulator mumu:MuMuPlayer-12.0-0", _backend.Calls); } + + [Fact] + public async Task Get_foreground_app_returns_package_activity_and_display_id() + { + await using var client = await ConnectAsync(); + + var r = await CallAsync(client, "get_foreground_app"); + + Assert.NotEqual(true, r.IsError); + var text = Text(r); + Assert.Contains("\"package\":\"com.demo.game\"", text, StringComparison.Ordinal); + Assert.Contains("\"component\":\"com.demo.game/.MainActivity\"", text, StringComparison.Ordinal); + Assert.Contains("foreground ", _backend.Calls); + } } diff --git a/tests/ClickZen.Platform.Tests/EmulatorDetectorTests.cs b/tests/ClickZen.Platform.Tests/EmulatorDetectorTests.cs index 2fd3079..9ac6baa 100644 --- a/tests/ClickZen.Platform.Tests/EmulatorDetectorTests.cs +++ b/tests/ClickZen.Platform.Tests/EmulatorDetectorTests.cs @@ -18,6 +18,9 @@ internal sealed class FakeEmulatorEnvironment : IEmulatorEnvironment /// (machine, key, value) → data. public Dictionary<(bool, string, string), string> Registry { get; } = []; + /// (exePath, args) → stdout. + public Dictionary<(string, string), string> Commands { get; } = []; + public Dictionary Environment { get; } = new(StringComparer.OrdinalIgnoreCase) { ["TEMP"] = @"C:\Users\me\AppData\Local\Temp", @@ -80,6 +83,9 @@ public string ExpandEnvironment(string text) return text; } + + public string? RunCommand(string fileName, string arguments, int timeoutMs = 1500) => + Commands.TryGetValue((fileName, arguments), out var stdout) ? stdout : null; } public sealed class EmulatorDetectorTests @@ -343,4 +349,89 @@ public void Executable_directory(string command, string? expected) { Assert.Equal(expected, EmulatorDetector.ExecutableDirectory(command)); } + + [Fact] + public void MuMu_v6_numeric_dash_v_index_without_instance_zero_or_vm_cmdline() + { + // Issue #4 scenario: instance 1 (127.0.0.1:16416) started via `-v 1`, instance 0 deleted, + // and MuMuVMMHeadless running under MuMuVMMSVC (SYSTEM) with unreadable command line. + const string root = @"D:\Program Files\Netease\MuMu"; + var env = new FakeEmulatorEnvironment(); + env.AddFile(root + @"\nx_main\MuMuNxMain.exe"); + env.AddFile(root + @"\vms\MuMuPlayer-12.0-1\configs\vm_config.json", + """{"vm":{"nat":{"port_forward":{"adb":{"host_port":"16416"}}}}}"""); + env.AddFile(root + @"\vms\MuMuPlayer-12.0-1\configs\extra_config.json", + """{"playerName":"MuMu模拟器12-1"}"""); + + var player = root + @"\nx_device\12.0\shell\MuMuNxDevice.exe"; + env.AddProcess(4100, player, $"\"{player}\" -v 1"); + env.AddProcess(4200, @"C:\Program Files\MuMuVMMVbox\Hypervisor\MuMuVMMHeadless.exe", ""); + env.AddWindow(0x101, 4100, "HintWnd", "Qt5156QWindowToolSaveBits"); + env.AddWindow(0x102, 4100, "MuMu安卓设备", "Qt5156QWindowIcon"); + + var instance = Assert.Single(new EmulatorDetector(env).Detect().Running); + Assert.Equal("MuMuPlayer-12.0-1", instance.InstanceKey); + Assert.Equal(1, instance.Index); + Assert.Equal((16416, AdbPortSource.Config), (instance.AdbPort, instance.PortSource)); + Assert.Equal((nint)0x102, instance.WindowHandle); + Assert.Equal("MuMu模拟器12-1", instance.Name); + Assert.True(instance.OwnsSerial("127.0.0.1:16416")); + } + + [Fact] + public void MuMu_elevated_with_relocated_vms_falls_back_to_MuMuManager_v6() + { + const string root = @"C:\Program Files\Netease\MuMuPlayer"; + var mgr = root + @"\nx_main\MuMuManager.exe"; + var player = root + @"\nx_device\15.0\shell\MuMuNxDevice.exe"; + + var env = new FakeEmulatorEnvironment(); + env.AddFile(mgr); + // Elevated process: command line could not be read (only executable path), and vms\ lives on another drive. + env.AddProcess(5100, player, $"\"{player}\""); + env.AddWindow(0x2A04, 5100, "MuMu安卓设备", "Qt6QWindowIcon"); + env.Commands[(mgr, "info --vmindex all")] = """ + { + "1": { + "index": "1", + "name": "明日方舟/塔塔", + "is_process_started": true, + "is_android_started": true, + "main_wnd": "2a04" + } + } + """; + env.Commands[(mgr, "adb --vmindex all")] = """ + { + "1": { "adb_host": "127.0.0.1", "adb_port": 16416 } + } + """; + + var instance = Assert.Single(new EmulatorDetector(env).Detect().Running); + Assert.Equal("MuMuPlayer-15.0-1", instance.InstanceKey); + Assert.Equal(1, instance.Index); + Assert.Equal((16416, AdbPortSource.Runtime), (instance.AdbPort, instance.PortSource)); + Assert.Equal("明日方舟/塔塔", instance.Name); + Assert.Equal((nint)0x2A04, instance.WindowHandle); + } + + [Fact] + public void MuMuExternalRenderer_locates_sdk_dll_and_flips_rgba_to_bgra() + { + const string root = @"D:\MuMu"; + var env = new FakeEmulatorEnvironment(); + var dll12 = root + @"\nx_device\12.0\shell\sdk\external_renderer_ipc.dll"; + var dll15 = root + @"\nx_device\15.0\shell\sdk\external_renderer_ipc.dll"; + env.AddFile(dll12); + Assert.Equal(dll12, MuMuExternalRenderer.FindSdkDll(root, env.GetDirectories, env.FileExists)); + + env.AddFile(dll15); + Assert.Equal(dll15, MuMuExternalRenderer.FindSdkDll(root, env.GetDirectories, env.FileExists)); + + // 1x2 bottom-up RGBA: bottom row = (R=10,G=20,B=30,A=255), top row = (R=40,G=50,B=60,A=128). + var rgba = new byte[] { 10, 20, 30, 255, 40, 50, 60, 128 }; + var bgra = MuMuExternalRenderer.ConvertBottomUpRgbaToTopDownBgra(rgba, width: 1, height: 2); + // Top-down BGRA: row 0 = top row (B=60,G=50,R=40,A=128), row 1 = bottom row (B=30,G=20,R=10,A=255). + Assert.Equal(new byte[] { 60, 50, 40, 128, 30, 20, 10, 255 }, bgra); + } }