From 66852efe18464a323e2730a57524e180d1cbec7e Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:59:33 -0700 Subject: [PATCH 1/2] A web page's range is a slider: named by its label, filled by position (CHALLENGES 319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit map printed MobiumApp's as "50 (input)", named by its value, and app_fill typed into it: fill 0.2, the position every native slider takes, was clamped to a step and failed as "the page rewrote or refused the value". A range is now a slider: map names it by its label and reads its value as its state, app_fill takes a position from 0 to 1 and sets the value from the page's own min, max and step — so it lands exactly, where a native iOS slider cannot (219) — and app_type is refused naming app_fill. From the native side, Android's WebView draws the range as a SeekBar whose text is its label and which reports no progress; it was mapped as "SeekBar" valued by its label. A seek bar inside a WebView keeps its text as its name, and app_fill on it goes through the page. docs/checks/slider-ios.sh asserts every kind of slider on MobiumApp's Slider Demo on iOS (318, 234, 319): 3 of 3 on the simulator, failing at the range with main's binary. slider.sh gains the web range on Android and a wait for the screen it maps, a race that had passed by timing (233): 3 of 3 on the emulator. --- docs/CHALLENGES.md | 35 ++++++ docs/ROADMAP.md | 2 +- docs/checks/README.md | 3 +- docs/checks/slider-ios.sh | 103 ++++++++++++++++++ docs/checks/slider.sh | 31 +++++- internal/agent/handlers.go | 8 ++ internal/agent/schema.go | 4 +- internal/agent/webrange_test.go | 65 +++++++++++ internal/agent/webview.go | 45 +++++++- internal/uitree/parse.go | 16 ++- .../testdata/mobiumapp-sliders-uia2.xml | 41 +++++++ internal/uitree/webseekbar_test.go | 37 +++++++ internal/webview/actionable.go | 73 ++++++++++++- internal/webview/actionable_test.go | 22 ++++ internal/webview/map.go | 13 ++- 15 files changed, 486 insertions(+), 12 deletions(-) create mode 100755 docs/checks/slider-ios.sh create mode 100644 internal/agent/webrange_test.go create mode 100644 internal/uitree/testdata/mobiumapp-sliders-uia2.xml create mode 100644 internal/uitree/webseekbar_test.go diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 2638952c..05ac9e3d 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -6832,6 +6832,41 @@ since it snaps to steps of 10; Balance, which is continuous, landed at 0.28 for 0.25, because a finger on the thumb is not the thumb's center, the inset Android's seek bars needed (227). A tap on the track moves nothing. +### 319. A web page's range was a text field to `map` and to `fill` + +**Found by:** giving MobiumApp's Slider Demo a web page's +`` (0 to 100 in steps of 10) beside its native sliders, +to measure every kind of slider on iOS (ROADMAP). `map` printed it as +`50 (input)` — named by its value, the mistake CHALLENGES 65, 219 and 234 +fixed natively — and `app_fill` typed into it: `fill 20` worked, while +`fill 0.2`, the position every native slider takes, was clamped by the +page to a step and failed as "the page rewrote or refused the value", a +cause that was not the case. `app_type` was accepted too, where a native +slider refuses it. **Fixed:** a range is a slider. `map` names it by its +label and reads its value as its state — `Range, 0 to 100 in steps of 10 +(slider, 50)` — and `role=slider` finds it. `app_fill` takes a position from +0 to 1, as on every other slider, and anything else is refused before +anything moves; `app_type` is refused naming `app_fill`. Unlike a native +slider (219), a page gives its range's `min`, `max` and `step`, so the +position is turned into the value the page would take and set through the +native setter with the `input` and `change` events a drag causes, and it +lands exactly: on the iPhone 17 Pro simulator 0.2, 1, 0, 0.35 and 0.5 read +20, 100, 0, 40 and 50, and the page's own output followed each. + +**And from the native side, on Android.** Android's WebView draws the range +as a `SeekBar` — but where a native seek bar's text is its progress (227), +Chrome's carries the range's label and no progress at all, so `map` in +`NATIVE_APP` printed `SeekBar (slider, Range, 0 to 100 in steps of 10 in its +own steps)`: named by its class and valued by its name. A seek bar inside a +WebView now keeps its text as its name and has no value, and `app_fill` on +it goes through the page when the page can be reached, as a field in a page +does (301) — on the Android 15 emulator, 0.3 read "30", through its page. +Adding these rows to `slider.sh` found it mapping the screen before the +Slider Demo had opened, with nothing waited for — CHALLENGES 233's race, +which had passed by timing; it waits for the screen now. `slider.sh` passed +three times in three on the emulator and `slider-ios.sh` three in three on +the simulator, which failed at the range with the binary before this fix. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 522aa1cb..6ff41bc7 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -948,7 +948,7 @@ this is what is not. a fill that lands exactly from partway, which WebDriverAgent cannot support — it reports no thumb position, and its range is real for a UIKit slider and 0 to 1 for SwiftUI's (CHALLENGES 219, 2026-10-13); - range sliders; and the phone's text + range sliders, two thumbs on one track; and the phone's text size, which is a slider in Settings. A custom control that does carry the Adjustable trait — Pocket Casts' scrubber — maps as `adjustable` with its value since 2026-10-05 (CHALLENGES 234), and `app_fill` refuses it, diff --git a/docs/checks/README.md b/docs/checks/README.md index f80f1f11..5abda931 100644 --- a/docs/checks/README.md +++ b/docs/checks/README.md @@ -92,7 +92,8 @@ checks do not: `clean-stop.sh`, which stops the caller's daemon, and | [pocketcasts-ios.sh](pocketcasts-ios.sh) | **A fourth third-party app on iOS**: Pocket Casts, a podcast player, signed out, built from its source for a simulator or from the App Store on a phone, and CHALLENGES 234–239 held against it: onboarding's button mapped disabled and `map --diff` saying when it is enabled, a row's checkmark not a target of its own, Discover's rows and a podcast's header named by their words rather than the buttons and images inside them, Play touched clear of the divider through its center, the player's first-run tip said by `map` and `app_alert` and closed where they say, and the scrubber mapped as adjustable with `app_fill` refused. On a simulator, iPhone or iPad, the app's data is cleared first; a phone's onboarding and tip run only on a fresh install, and the app is first brought back to Discover from wherever it was left. An episode is played for a moment and paused | | [kiwix-ios.sh](kiwix-ios.sh) | **A fifth third-party app on iOS, and the first whose WebView opens**: Kiwix, the offline Wikipedia reader, built from its source for a simulator or from the App Store on an iPhone, and CHALLENGES 244–248 and 256 held against it with a pinned ZIM: each catalog card one entry, off-screen ambiguity told to name one, an article tile one link named once, the disabled List button refused by its own ref, and a tap in the WebView — taller than its page, under the bars — opening the article it names; a page under the Library neither mapped nor tapped. A simulator starts from cleared data, with the ZIM uploaded and opened through the system document picker; a phone keeps its data, so the ZIM must already be in Kiwix's library from its own download, and 245 is reached only with no page under the Library | | [feed.sh](feed.sh) | **A list that grows as it is scrolled**, on MobiumApp's Feed Demo: twenty rows a page, the next 2.5 seconds after the end is reached, three pages in all. `scroll-to` reaches Row 55 across two loads instead of stopping at the first page's end, says Row 70 is past the real end at once, and an unknown role is refused rather than waited on (CHALLENGES 222, 223). Passed on the simulator, the iPhone and the Pixel 8 Pro | -| [slider.sh](slider.sh) | **Sliders on Android**, on MobiumApp's Slider Demo: Volume (0–100 in tens) and Balance (0–1, continuous) map as sliders by name with a value, are filled by position and land exactly — Volume on 30, 100 and 0, Balance on 0.25 — `map --diff` says the value moved, and a non-number and `type` are refused (CHALLENGES 227). Passed on the Pixel 8 Pro. Android only: on iOS the demo's sliders report no slider at all | +| [slider.sh](slider.sh) | **Sliders on Android**, on MobiumApp's Slider Demo: Volume (0–100 in tens) and Balance (0–1, continuous) map as sliders by name with a value, are filled by position and land exactly — Volume on 30, 100 and 0, Balance on 0.25 — `map --diff` says the value moved, and a non-number and `type` are refused (CHALLENGES 227). Passed on the Pixel 8 Pro. Android only: on iOS the demo is [slider-ios.sh](slider-ios.sh) | +| [slider-ios.sh](slider-ios.sh) | **Sliders on iOS, three kinds**, on MobiumApp's Slider Demo: Volume, a plain view to iOS (`map` calls it a button), where `fill` is refused as unsupported naming `swipe` and a drag lands it (CHALLENGES 318); Level, the same slider made accessible, mapped as adjustable with its value, `fill` refused naming a drag and the drag landing it (234); and a web page's range, a slider by its label, filled by position and landing exactly from the page's min, max and step, with a value and `type` refused (319). Passed 2026-10-13 three times in three on the iPhone 17 Pro simulator, about 25 seconds; with the binary before 319 it failed at the range | | [frames.sh](frames.sh) | **Frames inside a WebView**, on MobiumApp's Frames page: in the WebView's context every frame's elements are mapped with their frame — same-origin and nested through the page, cross-origin through its own execution context — each button tapped reaches its frame, the cross-origin card field is filled and added to as the frame reports, and `NATIVE_APP` reaches the cross-origin button too (CHALLENGES 228, 229). Passed on the iPhone simulator and an emulator | | [compose-app.sh](compose-app.sh) | **A Jetpack Compose app**, Seal from F-Droid, the first Compose app driven here, and CHALLENGES 176–178 held against it. Its icon buttons map once each as buttons. Its first-launch dialog, which the platform's alert endpoint does not know, is read by `alert` and not accepted, a tap on what it covers is refused naming it, and a rule answers it by its Close button. A switch row is set by its words, read back and put back, and a row with no state is refused. An app the check installed is uninstalled after, and on a phone nothing read off it is printed. Emulators and phones | | [map-diff.sh](map-diff.sh) | **`map --diff`**, on MobiumApp's Form Demo. The first diff of a session says it had nothing to compare with and adds the whole screen. A map with nothing done between says nothing changed, the control that it can come back empty. A box checked is its state and what it was, and on iOS, where its row grows, what is under it moves as one line, not five. A screen left is its elements gone and the next screen's added, and a ref from the diff acts. Emulators and simulators | diff --git a/docs/checks/slider-ios.sh b/docs/checks/slider-ios.sh new file mode 100755 index 00000000..eeae717a --- /dev/null +++ b/docs/checks/slider-ios.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Sliders on iOS, on MobiumApp's Slider Demo: three kinds, each handled the +# way the one fill rule says — a slider is filled with a position from 0 to +# 1, and a control Mobium cannot set to a position is refused, naming a drag. +# +# docs/checks/slider-ios.sh +# +# - Volume, @react-native-community/slider as shipped: on iOS a plain view +# with no trait and no value, so map calls it a button. app_fill is +# refused as unsupported, naming app_swipe, with nothing moved +# (CHALLENGES 318); a drag on its thumb lands it, in steps of 10. +# - Level, the same slider made accessible: the Adjustable trait and a +# value. Mapped as adjustable with its value; app_fill refused naming a +# drag (234); the drag lands it and map reads the new value. +# - Range, a web page's , 0 to 100 in steps of 10: a +# slider by its label, filled by position, landing exactly from the +# page's own min, max and step; a value and app_type refused (319). +# +# Android's sliders are slider.sh. Real UIKit and SwiftUI sliders, which +# app_fill moves (219), are icecubes-ios.sh. +set -e +DEV="$1" +if [ -z "$DEV" ]; then echo "usage: $0 " >&2; exit 2; fi +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +. "$ROOT/docs/checks/lib.sh" +check_platform ios "$DEV" +check_lock "$DEV" +M="$ROOT/bin/mobium --device $DEV" +APP=dev.mobium.mobiumapp +fail() { echo "FAIL: $*" >&2; exit 1; } +row() { printf ' %-10s %-60s ok\n' "$1" "$2"; } + +# refused : refused with that exit status, saying so. +refused() { + want=$1 words=$2; shift 2 + set +e; out=$("$@" 2>&1); got=$?; set -e + [ "$got" = "$want" ] || fail "$*: exit $got, want $want: $out" + echo "$out" | grep -q "$words" || fail "$*: refused without saying \"$words\": $out" +} +state() { $M text testid=sliderState; } +# The track's bounds in device pixels, from map: x1 x2 y. +track() { $M map --json | python3 -c ' +import json, sys +for e in json.load(sys.stdin)["elements"]: + if e["label"] == sys.argv[1]: + b = e["bounds"]; print(b["x1"], b["x2"], (b["y1"] + b["y2"]) // 2); break' "$1"; } +# drag