diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index cf5522a..ce5896e 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -6634,6 +6634,42 @@ view, as it already did before any web action touched an element, in a cross-origin frame's own context too, and the answer gives the element's place on the screen. No direction is needed there. +### 310. A search up past the start of a list refreshed the app, and said nothing + +**Found by:** asking what `scroll-to` still got wrong, then giving +MobiumApp's Feed Demo pull to refresh, counted on its status line. A +scroll loop ends a list with a swipe that moves nothing, and at the start +of a list that swipe is a pull. From Row 45, sixty rows loaded, `scroll-to +text=Nowhere --direction up` reached the top in eight swipes and answered +"the end of the list": the feed had refreshed twice on the iPhone 15 Plus +and the simulator and once on the Android 15 emulator, and its sixty rows +were twenty. The loading wait (223) made it worse on iOS: the refresh +spinner read as rows on their way, was waited out, and the next swipe +pulled again. In a real app the pull is a network reload, a reset scroll +position and lost pages, and nothing said so. + +What can be prevented, measured: on the simulator only a swipe past the +start pulls, and the one that reaches it coasts there without pulling, so +stopping before the extra swipe is enough. Its sign is a swipe that moved +the rows clearly less than the one before it — or not at all while +something else moved, the spinner — with the first row whole at the edge; +against the swipe before, because a simulator's swipe coasts to twice its +length, and the one that reached the top, 747 pixels against 1,617, was no +shorter than the finger. On Android and the iPhone the swipe that reaches +the start pulls already, three times in three on the emulator from 300 +pixels down, so nothing short of knowing where the start is could spare +that one, and a list does not say. **Fixed:** the loop stops at the start +instead of swiping again; a busy indicator that came and went with nothing +new is no longer more rows, toward the start or the end; and a search that +reaches the start says the last swipe there is a pull an app may answer by +refreshing, in its text and as `may_have_refreshed` in its details. From +Row 45: the simulator six swipes, no refresh, sixty rows kept; the iPhone +six and one, against eight and two before; the Pixel 8 Pro (Android 17) +six and none, sixty rows kept, against seven and two — its swipe that +reaches the start does not pull, as the emulator's does; the emulator +eight and one, as before, and said. `feed.sh` asserts the warning and at +most one pull, and passed on all four. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/checks/feed.sh b/docs/checks/feed.sh index 68dc5b9..3577bd8 100755 --- a/docs/checks/feed.sh +++ b/docs/checks/feed.sh @@ -20,7 +20,11 @@ # the negative control: a real end must still read as one; # - a role nothing has is refused, rather than matching nothing: the help's # own `wait role=progressbar --for hidden` was over at once before -# progressbar was a role (CHALLENGES 222). +# progressbar was a role (CHALLENGES 222); +# - a search back up past the start says its last swipe may have been a +# pull the app answered by refreshing, and pulls at most once +# (CHALLENGES 310) — counted by the Feed's own refreshes, which needs +# MobiumApp with pull to refresh. # # Passed on the simulator, the iPhone 15 Plus and the Pixel 8 Pro, where the # spinner is an android.widget.ProgressBar. @@ -65,5 +69,21 @@ out=$($M wait role=nosuchrole --for hidden --timeout 1s 2>&1) && fail "a role no echo "$out" | grep -q "unknown role" || fail "an unknown role was refused without saying why: $out" row "role" "an unknown role refused, naming the roles there are" +# Back up past the start, from deep in the loaded feed. At the start of a +# list a swipe is a pull, and the Feed refreshes on one: the search said +# only "the end of the list" while it pulled twice on an iPhone and a +# simulator and threw the loaded rows away (CHALLENGES 310). Now it says the +# last swipe may have refreshed, and pulls at most once — the swipe that +# reaches the start, which on Android and a real iPhone pulls already. +refreshes() { $M text testid=feedState | sed -n 's/.*refreshes: \([0-9]*\).*/\1/p'; } +before=$(refreshes); before=${before:-0} +out=$($M scroll-to "text=Nowhere" --direction up 2>&1) && fail "found what is not in the feed: $out" +echo "$out" | grep -q "start of the list" || fail "the search up did not say it reached the start: $out" +echo "$out" | grep -q "refreshing" || fail "the search up did not say its last swipe may have refreshed: $out" +sleep 2 +after=$(refreshes); after=${after:-0} +[ $(( after - before )) -le 1 ] || fail "one search up refreshed the feed $(( after - before )) times" +row "start" "up past the start: said, and at most one pull" + $M terminate "$APP" >/dev/null 2>&1 || true echo PASS diff --git a/internal/agent/scroll.go b/internal/agent/scroll.go index 1f6c47a..0a613ef 100644 --- a/internal/agent/scroll.go +++ b/internal/agent/scroll.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/mobiumdev/mobium/internal/mobiumerr" "math" + "sort" "strings" "time" @@ -270,6 +271,7 @@ func (h *Handlers) scrollIntoView(ctx context.Context, s *session, loc uitree.Lo // button in plain sight above it. CHALLENGES 114. partial, partialIn := offScreenTarget(pick, tree) + lastShift := 0 // how far the last full swipe moved the rows for i := 1; i <= maxScrolls; i++ { nudged, dx, dy, err := nudgeInto(ctx, gest, partialIn, partial, horizontal(dir), followTarget) if err != nil { @@ -317,6 +319,32 @@ func (h *Handlers) scrollIntoView(ctx context.Context, s *session, loc uitree.Lo partial, partialIn = offScreenTarget(pick, tree) now := read(container) + // Toward the start, a swipe that moved the rows clearly less than the + // one before it — or not at all, while something else moved, such as + // a refresh spinner — leaving the first row whole at the edge, has + // reached the start: one more swipe would only prove it, and at the + // start of a list that swipe is a pull. Against the swipe before, + // not the swipe's own length: an iPhone simulator's swipe coasts to + // twice its length, so the one that reached the top partway, 747 + // pixels against 1,617 before it, was no shorter than the finger. + // On MobiumApp's Feed Demo the extra swipe refreshed the feed and + // threw away the rows it had loaded. CHALLENGES 310. + d, measured := shift(prev, now, horizontal(dir)) + if !nudged && towardStart(dir) && moved(prev, now) && measured && d >= 0 { + base := lastShift + if base <= 0 { + base = container.Bounds.Height() / 2 + if horizontal(dir) { + base = container.Bounds.Width() / 2 + } + } + if d*10 < base*6 && atStart(container, horizontal(dir)) { + return nil, nil, i, endOfList(loc, resolveErr, dir, i, true) + } + } + if !nudged && measured && d > 0 { + lastShift = d + } if !moved(prev, now) { // Not the end while the list is still loading what follows. A // feed fetches its next page when its end is reached, and the @@ -342,11 +370,18 @@ func (h *Handlers) scrollIntoView(ctx context.Context, s *session, loc uitree.Lo return nil, nil, i, mobiumerr.New(mobiumerr.Timeout, "%s — scrolled %s to where the list "+ "loads more, and it was still loading after %s", whyNot(loc, resolveErr), dir, loadWait) } - prev = read(container) + after := read(container) + // Nothing new: the indicator was not more rows on their way. + // Toward the start it was the app refreshing on the pull this + // swipe was, and swiping on pulled again — twice on an iOS + // simulator, for one search. CHALLENGES 310. + if !moved(now, after) { + return nil, nil, i, endOfList(loc, resolveErr, dir, i, towardStart(dir)) + } + prev = after continue } - return nil, nil, i, mobiumerr.New(mobiumerr.NoSuchElement, "%s — scrolled %s to the end of the list "+ - "(%d scroll%s) without bringing it into view", whyNot(loc, resolveErr), dir, i, plural(i)) + return nil, nil, i, endOfList(loc, resolveErr, dir, i, false) } prev = now } @@ -827,6 +862,9 @@ type reading struct { geometry string // texts is what everything says, in document order. texts []string + // at is where each child that says something unique is, to measure how + // far a swipe moved the list. + at map[string]uitree.Rect } // read captures a container for later comparison. @@ -840,21 +878,101 @@ func read(container *uitree.Node) reading { if len(container.Children) == 0 && container.Parent != nil { container = container.Parent } - var r reading + r := reading{at: map[string]uitree.Rect{}} + seen := map[string]int{} var b strings.Builder var visit func(*uitree.Node) visit = func(n *uitree.Node) { for _, c := range n.Children { fmt.Fprintf(&b, "%s\n", c.Bounds) - r.texts = append(r.texts, c.Text+"\x00"+c.Label) + key := c.Text + "\x00" + c.Label + r.texts = append(r.texts, key) + if key != "\x00" && !c.Bounds.Empty() { + seen[key]++ + r.at[key] = c.Bounds + } visit(c) } } visit(container) + for k, n := range seen { + if n > 1 { + delete(r.at, k) + } + } r.geometry = b.String() return r } +// shift is how far the list moved between two readings along the axis, the +// middle of what the children that said the same thing in both moved; ok is +// false with fewer than three to go by. +func shift(before, after reading, horizontal bool) (int, bool) { + var ds []int + for k, a := range before.at { + if b, ok := after.at[k]; ok { + if horizontal { + ds = append(ds, b.X1-a.X1) + } else { + ds = append(ds, b.Y1-a.Y1) + } + } + } + if len(ds) < 3 { + return 0, false + } + sort.Ints(ds) + return ds[len(ds)/2], true +} + +// atStart says the list shows its first row whole, at its top or below it — +// or its left edge, scrolling sideways: the first child is not past that +// edge and is as tall (or wide) as the one after it. Below it, because a +// list refreshing pushes its rows down under its spinner (iOS, measured on +// MobiumApp's Feed Demo). Android clips a row cut by its list to the list, +// so flush alone would be true of any row scrolled halfway past the top; +// whole is the second sign (CHALLENGES 169). Used only with a swipe that +// moved the list short, never alone: a row that happens to rest flush in +// the middle of a list looks the same. +func atStart(container *uitree.Node, horizontal bool) bool { + // Down through the wrappers to the rows: an only child, scroll bars + // aside — an iOS scroll view holds its bars beside the one wrapper + // (CHALLENGES 307). + rows := func(n *uitree.Node) []*uitree.Node { + var out []*uitree.Node + for _, k := range n.Children { + if !k.Bounds.Empty() && !uitree.IsScrollIndicator(k) { + out = append(out, k) + } + } + return out + } + kids := rows(container) + for len(kids) == 1 { + kids = rows(kids[0]) + } + if len(kids) < 2 { + return false + } + uitree.ReadingOrder(kids) + first, next := kids[0].Bounds, kids[1].Bounds + if horizontal { + return first.X1 >= container.Bounds.X1-2 && abs(first.Width()-next.Width()) <= 2 + } + return first.Y1 >= container.Bounds.Y1-2 && abs(first.Height()-next.Height()) <= 2 +} + +func abs(x int) int { + if x < 0 { + return -x + } + return x +} + +// towardStart is scrolling up or left, where a list begins — and where a +// swipe past its start is a pull to refresh. +func towardStart(dir string) bool { return dir == "up" || dir == "left" } + // moved reports whether the list actually went anywhere between two readings. // // Text alone is not evidence. The About screen on a Pixel 7 shows an uptime @@ -940,3 +1058,24 @@ func (h *Handlers) scrollToWeb(ctx context.Context, s *session, target string) ( } return Result(msg, view), nil } + +// endOfList is the miss at the end of a list. Toward the start it says what +// the last swipe may have done: at the start of a list a swipe is a pull, +// and an app may answer it by refreshing — MobiumApp's Feed Demo reloaded, +// losing forty rows it had loaded. On Android the swipe that reaches the +// start pulls already, three times in three on the emulator, so nothing +// short of knowing where the start is could spare it; on iOS only a swipe +// past the start pulls, and the loop now stops before one. Whether a pull +// was answered cannot be read from outside — on iOS nothing of the refresh +// is in the tree once the screen is read — so it is said as what may have +// happened, in the text and the structured answer both. CHALLENGES 310. +func endOfList(loc uitree.Locator, resolveErr error, dir string, scrolls int, _ bool) error { + if !towardStart(dir) { + return mobiumerr.New(mobiumerr.NoSuchElement, "%s — scrolled %s to the end of the list (%d scroll%s) without "+ + "bringing it into view", whyNot(loc, resolveErr), dir, scrolls, plural(scrolls)).WithDetail("scrolls", scrolls) + } + return mobiumerr.New(mobiumerr.NoSuchElement, "%s — scrolled %s to the start of the list (%d scroll%s) without "+ + "bringing it into view; at the start of a list the last swipe is a pull, which an app may answer by "+ + "refreshing what it shows", whyNot(loc, resolveErr), dir, scrolls, plural(scrolls)). + WithDetail("scrolls", scrolls).WithDetail("may_have_refreshed", true) +} diff --git a/internal/agent/scroll_test.go b/internal/agent/scroll_test.go index e97e6a8..6dfea3c 100644 --- a/internal/agent/scroll_test.go +++ b/internal/agent/scroll_test.go @@ -953,3 +953,37 @@ func TestTheSwipeLimitSaysToGoOn(t *testing.T) { } t.Error("scroll-to again never went on to Row 399") } + +// How far a list moved is the middle of what its named rows moved; fewer +// than three to go by is no measurement. +func TestShiftIsWhatTheRowsMoved(t *testing.T) { + r := func(ys ...int) reading { + at := map[string]uitree.Rect{} + for i, y := range ys { + at[fmt.Sprintf("Row %d", i)] = uitree.Rect{X1: 0, Y1: y, X2: 100, Y2: y + 50} + } + return reading{at: at} + } + if d, ok := shift(r(0, 60, 120), r(300, 360, 420), false); !ok || d != 300 { + t.Errorf("shift = %d, %v; want 300", d, ok) + } + if _, ok := shift(r(0, 60), r(300, 360), false); ok { + t.Error("two rows were taken for a measurement") + } +} + +// Toward the start, the miss says the last swipe may have been a pull an +// app answered by refreshing, in the text and the structured answer; toward +// the end it says nothing of the kind. CHALLENGES 310. +func TestTheStartOfAListSaysItMayHaveRefreshed(t *testing.T) { + loc, _ := uitree.ParseLocator("text=Nowhere") + up, _ := mobiumerr.As(endOfList(loc, nil, "up", 6, false)) + if !strings.Contains(up.Error(), "start of the list") || !strings.Contains(up.Error(), "refreshing") || + up.Details["may_have_refreshed"] != true { + t.Errorf("up: %v %v", up, up.Details) + } + down, _ := mobiumerr.As(endOfList(loc, nil, "down", 3, false)) + if !strings.Contains(down.Error(), "end of the list") || down.Details["may_have_refreshed"] != nil { + t.Errorf("down: %v %v", down, down.Details) + } +} diff --git a/internal/uitree/cover.go b/internal/uitree/cover.go index c716a57..647ae2f 100644 --- a/internal/uitree/cover.go +++ b/internal/uitree/cover.go @@ -74,6 +74,10 @@ func isScrollIndicator(n *Node) bool { return n.Class == "XCUIElementTypeOther" && !n.Clickable && percent.MatchString(n.Text) } +// IsScrollIndicator is an iOS scroll view's scroll bar: a child of the +// list, never one of its rows. +func IsScrollIndicator(n *Node) bool { return isScrollIndicator(n) } + var percent = regexp.MustCompile(`^\d{1,3}%$`) func isIOSClass(class string) bool {