From 88243fd30a37c9fbfacf79df63b4d3d495c02d51 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:30:43 -0700 Subject: [PATCH] scroll-to: up past the start pulls at most once and says it may have refreshed (CHALLENGES 310) At the start of a list the swipe that proves the end is a pull: from a fed list sixty rows deep, a search up refreshed MobiumApp's Feed twice on the iPhone and the simulator and once on the Android emulator, losing the rows, and said only "the end of the list". The loop now stops at the start once a swipe moved the rows clearly less than the one before, the first row whole at the edge; a busy indicator that came and went with nothing new is not more rows; and a search that reaches the start says the last swipe may have been answered by a refresh, in text and details. Measured: simulator no refresh and rows kept; iPhone one pull, from two; emulator one, which the swipe reaching the start makes there. feed.sh asserts it on all three. --- docs/CHALLENGES.md | 36 ++++++++ docs/checks/feed.sh | 22 ++++- internal/agent/scroll.go | 149 ++++++++++++++++++++++++++++++++-- internal/agent/scroll_test.go | 34 ++++++++ internal/uitree/cover.go | 4 + 5 files changed, 239 insertions(+), 6 deletions(-) diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index cf5522a..ce5896e 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -6634,6 +6634,42 @@ view, as it already did before any web action touched an element, in a cross-origin frame's own context too, and the answer gives the element's place on the screen. No direction is needed there. +### 310. A search up past the start of a list refreshed the app, and said nothing + +**Found by:** asking what `scroll-to` still got wrong, then giving +MobiumApp's Feed Demo pull to refresh, counted on its status line. A +scroll loop ends a list with a swipe that moves nothing, and at the start +of a list that swipe is a pull. From Row 45, sixty rows loaded, `scroll-to +text=Nowhere --direction up` reached the top in eight swipes and answered +"the end of the list": the feed had refreshed twice on the iPhone 15 Plus +and the simulator and once on the Android 15 emulator, and its sixty rows +were twenty. The loading wait (223) made it worse on iOS: the refresh +spinner read as rows on their way, was waited out, and the next swipe +pulled again. In a real app the pull is a network reload, a reset scroll +position and lost pages, and nothing said so. + +What can be prevented, measured: on the simulator only a swipe past the +start pulls, and the one that reaches it coasts there without pulling, so +stopping before the extra swipe is enough. Its sign is a swipe that moved +the rows clearly less than the one before it — or not at all while +something else moved, the spinner — with the first row whole at the edge; +against the swipe before, because a simulator's swipe coasts to twice its +length, and the one that reached the top, 747 pixels against 1,617, was no +shorter than the finger. On Android and the iPhone the swipe that reaches +the start pulls already, three times in three on the emulator from 300 +pixels down, so nothing short of knowing where the start is could spare +that one, and a list does not say. **Fixed:** the loop stops at the start +instead of swiping again; a busy indicator that came and went with nothing +new is no longer more rows, toward the start or the end; and a search that +reaches the start says the last swipe there is a pull an app may answer by +refreshing, in its text and as `may_have_refreshed` in its details. From +Row 45: the simulator six swipes, no refresh, sixty rows kept; the iPhone +six and one, against eight and two before; the Pixel 8 Pro (Android 17) +six and none, sixty rows kept, against seven and two — its swipe that +reaches the start does not pull, as the emulator's does; the emulator +eight and one, as before, and said. `feed.sh` asserts the warning and at +most one pull, and passed on all four. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/checks/feed.sh b/docs/checks/feed.sh index 68dc5b9..3577bd8 100755 --- a/docs/checks/feed.sh +++ b/docs/checks/feed.sh @@ -20,7 +20,11 @@ # the negative control: a real end must still read as one; # - a role nothing has is refused, rather than matching nothing: the help's # own `wait role=progressbar --for hidden` was over at once before -# progressbar was a role (CHALLENGES 222). +# progressbar was a role (CHALLENGES 222); +# - a search back up past the start says its last swipe may have been a +# pull the app answered by refreshing, and pulls at most once +# (CHALLENGES 310) — counted by the Feed's own refreshes, which needs +# MobiumApp with pull to refresh. # # Passed on the simulator, the iPhone 15 Plus and the Pixel 8 Pro, where the # spinner is an android.widget.ProgressBar. @@ -65,5 +69,21 @@ out=$($M wait role=nosuchrole --for hidden --timeout 1s 2>&1) && fail "a role no echo "$out" | grep -q "unknown role" || fail "an unknown role was refused without saying why: $out" row "role" "an unknown role refused, naming the roles there are" +# Back up past the start, from deep in the loaded feed. At the start of a +# list a swipe is a pull, and the Feed refreshes on one: the search said +# only "the end of the list" while it pulled twice on an iPhone and a +# simulator and threw the loaded rows away (CHALLENGES 310). Now it says the +# last swipe may have refreshed, and pulls at most once — the swipe that +# reaches the start, which on Android and a real iPhone pulls already. +refreshes() { $M text testid=feedState | sed -n 's/.*refreshes: \([0-9]*\).*/\1/p'; } +before=$(refreshes); before=${before:-0} +out=$($M scroll-to "text=Nowhere" --direction up 2>&1) && fail "found what is not in the feed: $out" +echo "$out" | grep -q "start of the list" || fail "the search up did not say it reached the start: $out" +echo "$out" | grep -q "refreshing" || fail "the search up did not say its last swipe may have refreshed: $out" +sleep 2 +after=$(refreshes); after=${after:-0} +[ $(( after - before )) -le 1 ] || fail "one search up refreshed the feed $(( after - before )) times" +row "start" "up past the start: said, and at most one pull" + $M terminate "$APP" >/dev/null 2>&1 || true echo PASS diff --git a/internal/agent/scroll.go b/internal/agent/scroll.go index 1f6c47a..0a613ef 100644 --- a/internal/agent/scroll.go +++ b/internal/agent/scroll.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/mobiumdev/mobium/internal/mobiumerr" "math" + "sort" "strings" "time" @@ -270,6 +271,7 @@ func (h *Handlers) scrollIntoView(ctx context.Context, s *session, loc uitree.Lo // button in plain sight above it. CHALLENGES 114. partial, partialIn := offScreenTarget(pick, tree) + lastShift := 0 // how far the last full swipe moved the rows for i := 1; i <= maxScrolls; i++ { nudged, dx, dy, err := nudgeInto(ctx, gest, partialIn, partial, horizontal(dir), followTarget) if err != nil { @@ -317,6 +319,32 @@ func (h *Handlers) scrollIntoView(ctx context.Context, s *session, loc uitree.Lo partial, partialIn = offScreenTarget(pick, tree) now := read(container) + // Toward the start, a swipe that moved the rows clearly less than the + // one before it — or not at all, while something else moved, such as + // a refresh spinner — leaving the first row whole at the edge, has + // reached the start: one more swipe would only prove it, and at the + // start of a list that swipe is a pull. Against the swipe before, + // not the swipe's own length: an iPhone simulator's swipe coasts to + // twice its length, so the one that reached the top partway, 747 + // pixels against 1,617 before it, was no shorter than the finger. + // On MobiumApp's Feed Demo the extra swipe refreshed the feed and + // threw away the rows it had loaded. CHALLENGES 310. + d, measured := shift(prev, now, horizontal(dir)) + if !nudged && towardStart(dir) && moved(prev, now) && measured && d >= 0 { + base := lastShift + if base <= 0 { + base = container.Bounds.Height() / 2 + if horizontal(dir) { + base = container.Bounds.Width() / 2 + } + } + if d*10 < base*6 && atStart(container, horizontal(dir)) { + return nil, nil, i, endOfList(loc, resolveErr, dir, i, true) + } + } + if !nudged && measured && d > 0 { + lastShift = d + } if !moved(prev, now) { // Not the end while the list is still loading what follows. A // feed fetches its next page when its end is reached, and the @@ -342,11 +370,18 @@ func (h *Handlers) scrollIntoView(ctx context.Context, s *session, loc uitree.Lo return nil, nil, i, mobiumerr.New(mobiumerr.Timeout, "%s — scrolled %s to where the list "+ "loads more, and it was still loading after %s", whyNot(loc, resolveErr), dir, loadWait) } - prev = read(container) + after := read(container) + // Nothing new: the indicator was not more rows on their way. + // Toward the start it was the app refreshing on the pull this + // swipe was, and swiping on pulled again — twice on an iOS + // simulator, for one search. CHALLENGES 310. + if !moved(now, after) { + return nil, nil, i, endOfList(loc, resolveErr, dir, i, towardStart(dir)) + } + prev = after continue } - return nil, nil, i, mobiumerr.New(mobiumerr.NoSuchElement, "%s — scrolled %s to the end of the list "+ - "(%d scroll%s) without bringing it into view", whyNot(loc, resolveErr), dir, i, plural(i)) + return nil, nil, i, endOfList(loc, resolveErr, dir, i, false) } prev = now } @@ -827,6 +862,9 @@ type reading struct { geometry string // texts is what everything says, in document order. texts []string + // at is where each child that says something unique is, to measure how + // far a swipe moved the list. + at map[string]uitree.Rect } // read captures a container for later comparison. @@ -840,21 +878,101 @@ func read(container *uitree.Node) reading { if len(container.Children) == 0 && container.Parent != nil { container = container.Parent } - var r reading + r := reading{at: map[string]uitree.Rect{}} + seen := map[string]int{} var b strings.Builder var visit func(*uitree.Node) visit = func(n *uitree.Node) { for _, c := range n.Children { fmt.Fprintf(&b, "%s\n", c.Bounds) - r.texts = append(r.texts, c.Text+"\x00"+c.Label) + key := c.Text + "\x00" + c.Label + r.texts = append(r.texts, key) + if key != "\x00" && !c.Bounds.Empty() { + seen[key]++ + r.at[key] = c.Bounds + } visit(c) } } visit(container) + for k, n := range seen { + if n > 1 { + delete(r.at, k) + } + } r.geometry = b.String() return r } +// shift is how far the list moved between two readings along the axis, the +// middle of what the children that said the same thing in both moved; ok is +// false with fewer than three to go by. +func shift(before, after reading, horizontal bool) (int, bool) { + var ds []int + for k, a := range before.at { + if b, ok := after.at[k]; ok { + if horizontal { + ds = append(ds, b.X1-a.X1) + } else { + ds = append(ds, b.Y1-a.Y1) + } + } + } + if len(ds) < 3 { + return 0, false + } + sort.Ints(ds) + return ds[len(ds)/2], true +} + +// atStart says the list shows its first row whole, at its top or below it — +// or its left edge, scrolling sideways: the first child is not past that +// edge and is as tall (or wide) as the one after it. Below it, because a +// list refreshing pushes its rows down under its spinner (iOS, measured on +// MobiumApp's Feed Demo). Android clips a row cut by its list to the list, +// so flush alone would be true of any row scrolled halfway past the top; +// whole is the second sign (CHALLENGES 169). Used only with a swipe that +// moved the list short, never alone: a row that happens to rest flush in +// the middle of a list looks the same. +func atStart(container *uitree.Node, horizontal bool) bool { + // Down through the wrappers to the rows: an only child, scroll bars + // aside — an iOS scroll view holds its bars beside the one wrapper + // (CHALLENGES 307). + rows := func(n *uitree.Node) []*uitree.Node { + var out []*uitree.Node + for _, k := range n.Children { + if !k.Bounds.Empty() && !uitree.IsScrollIndicator(k) { + out = append(out, k) + } + } + return out + } + kids := rows(container) + for len(kids) == 1 { + kids = rows(kids[0]) + } + if len(kids) < 2 { + return false + } + uitree.ReadingOrder(kids) + first, next := kids[0].Bounds, kids[1].Bounds + if horizontal { + return first.X1 >= container.Bounds.X1-2 && abs(first.Width()-next.Width()) <= 2 + } + return first.Y1 >= container.Bounds.Y1-2 && abs(first.Height()-next.Height()) <= 2 +} + +func abs(x int) int { + if x < 0 { + return -x + } + return x +} + +// towardStart is scrolling up or left, where a list begins — and where a +// swipe past its start is a pull to refresh. +func towardStart(dir string) bool { return dir == "up" || dir == "left" } + // moved reports whether the list actually went anywhere between two readings. // // Text alone is not evidence. The About screen on a Pixel 7 shows an uptime @@ -940,3 +1058,24 @@ func (h *Handlers) scrollToWeb(ctx context.Context, s *session, target string) ( } return Result(msg, view), nil } + +// endOfList is the miss at the end of a list. Toward the start it says what +// the last swipe may have done: at the start of a list a swipe is a pull, +// and an app may answer it by refreshing — MobiumApp's Feed Demo reloaded, +// losing forty rows it had loaded. On Android the swipe that reaches the +// start pulls already, three times in three on the emulator, so nothing +// short of knowing where the start is could spare it; on iOS only a swipe +// past the start pulls, and the loop now stops before one. Whether a pull +// was answered cannot be read from outside — on iOS nothing of the refresh +// is in the tree once the screen is read — so it is said as what may have +// happened, in the text and the structured answer both. CHALLENGES 310. +func endOfList(loc uitree.Locator, resolveErr error, dir string, scrolls int, _ bool) error { + if !towardStart(dir) { + return mobiumerr.New(mobiumerr.NoSuchElement, "%s — scrolled %s to the end of the list (%d scroll%s) without "+ + "bringing it into view", whyNot(loc, resolveErr), dir, scrolls, plural(scrolls)).WithDetail("scrolls", scrolls) + } + return mobiumerr.New(mobiumerr.NoSuchElement, "%s — scrolled %s to the start of the list (%d scroll%s) without "+ + "bringing it into view; at the start of a list the last swipe is a pull, which an app may answer by "+ + "refreshing what it shows", whyNot(loc, resolveErr), dir, scrolls, plural(scrolls)). + WithDetail("scrolls", scrolls).WithDetail("may_have_refreshed", true) +} diff --git a/internal/agent/scroll_test.go b/internal/agent/scroll_test.go index e97e6a8..6dfea3c 100644 --- a/internal/agent/scroll_test.go +++ b/internal/agent/scroll_test.go @@ -953,3 +953,37 @@ func TestTheSwipeLimitSaysToGoOn(t *testing.T) { } t.Error("scroll-to again never went on to Row 399") } + +// How far a list moved is the middle of what its named rows moved; fewer +// than three to go by is no measurement. +func TestShiftIsWhatTheRowsMoved(t *testing.T) { + r := func(ys ...int) reading { + at := map[string]uitree.Rect{} + for i, y := range ys { + at[fmt.Sprintf("Row %d", i)] = uitree.Rect{X1: 0, Y1: y, X2: 100, Y2: y + 50} + } + return reading{at: at} + } + if d, ok := shift(r(0, 60, 120), r(300, 360, 420), false); !ok || d != 300 { + t.Errorf("shift = %d, %v; want 300", d, ok) + } + if _, ok := shift(r(0, 60), r(300, 360), false); ok { + t.Error("two rows were taken for a measurement") + } +} + +// Toward the start, the miss says the last swipe may have been a pull an +// app answered by refreshing, in the text and the structured answer; toward +// the end it says nothing of the kind. CHALLENGES 310. +func TestTheStartOfAListSaysItMayHaveRefreshed(t *testing.T) { + loc, _ := uitree.ParseLocator("text=Nowhere") + up, _ := mobiumerr.As(endOfList(loc, nil, "up", 6, false)) + if !strings.Contains(up.Error(), "start of the list") || !strings.Contains(up.Error(), "refreshing") || + up.Details["may_have_refreshed"] != true { + t.Errorf("up: %v %v", up, up.Details) + } + down, _ := mobiumerr.As(endOfList(loc, nil, "down", 3, false)) + if !strings.Contains(down.Error(), "end of the list") || down.Details["may_have_refreshed"] != nil { + t.Errorf("down: %v %v", down, down.Details) + } +} diff --git a/internal/uitree/cover.go b/internal/uitree/cover.go index c716a57..647ae2f 100644 --- a/internal/uitree/cover.go +++ b/internal/uitree/cover.go @@ -74,6 +74,10 @@ func isScrollIndicator(n *Node) bool { return n.Class == "XCUIElementTypeOther" && !n.Clickable && percent.MatchString(n.Text) } +// IsScrollIndicator is an iOS scroll view's scroll bar: a child of the +// list, never one of its rows. +func IsScrollIndicator(n *Node) bool { return isScrollIndicator(n) } + var percent = regexp.MustCompile(`^\d{1,3}%$`) func isIOSClass(class string) bool {