From 0c3f584c121363e0a5dbfd8130d46ced32884367 Mon Sep 17 00:00:00 2001 From: Espen Hovlandsdal Date: Mon, 7 Sep 2026 09:52:25 -0400 Subject: [PATCH 1/2] test: add iOS tests --- .github/workflows/test.yml | 16 ++++ CONTRIBUTING.md | 7 +- package.json | 1 + test/helpers/iosSimulator.ts | 168 +++++++++++++++++++++++++++++++++++ vitest.ios.config.ts | 32 +++++++ 5 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 test/helpers/iosSimulator.ts create mode 100644 vitest.ios.config.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb78302..1acb5d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -136,3 +136,19 @@ jobs: run: npm ci - name: Run tests run: npm run test:workerd + + testIos: + name: 'Test: iOS Simulator' + timeout-minutes: 20 + runs-on: macos-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + - name: Install dependencies + run: npm ci + - name: List available simulators + run: xcrun simctl list devices available + - name: Run tests + run: npm run test:ios diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2dec93..757d51b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,15 +26,20 @@ The suite in `test/client.test.ts` runs against a real HTTP server in every supp - `npm test` - Node.js - `npm run test:browser` - Chromium, Firefox and WebKit, via Playwright +- `npm run test:ios` - Safari on an iOS simulator (macOS with Xcode only) - `npm run test:bun` - Bun - `npm run test:deno` - Deno - `npm run test:happy-dom` - happy-dom - `npm run test:workerd` - workerd (Cloudflare Workers), via miniflare - `npm run test:types` - type compatibility with the WhatWG `EventSource` -- `npm run test:all` - all of the above, in sequence +- `npm run test:all` - all of the above except `test:ios`, in sequence The browser tests need Playwright's browsers installed once, with `npx playwright install chromium firefox webkit`. +`test:ios` is out of `test:all` because it only runs on macOS with Xcode's iOS platform installed, but do reach for it whenever a report names an iPhone or iPad: Playwright's `webkit` is a desktop build with a different networking stack, so it is not evidence about iOS. It picks an iPhone on the newest installed iOS runtime, preferring one that is already booted; set `IOS_SIMULATOR_DEVICE` to pin a device by name, eg `IOS_SIMULATOR_DEVICE='iPad (A16)' npm run test:ios`. If nothing is installed, `xcodebuild -downloadPlatform iOS` fetches a runtime. The simulator is left booted afterwards, since booting one costs most of a minute. + +The provider (`test/helpers/iosSimulator.ts`) drives the simulator with `simctl` alone - Safari there is opened with `simctl openurl` and reports back over the websocket Vitest already opens, so no WebDriver or Appium stack is involved. That also means nothing in the suite can drive the page: keep the tests free of Vitest's browser locators and `userEvent`, which this provider does not implement. + Every environment gates CI. Two of them deviate from the rest in ways that are the runtime's doing rather than ours, and those tests are asserted as known failures with `test.fails` rather than skipped or excluded, so each one turns red - prompting removal of the workaround - once the runtime is fixed: - **workerd**, seven of the `on*` handler tests. Its `EventTarget` dispatches `on` handler properties itself, on top of the `addEventListener` call our `on*` setters make, so those handlers fire more than once, assigning `null` only removes one registration, and they fire ahead of listeners registered before them ([cloudflare/workerd#6022](https://github.com/cloudflare/workerd/issues/6022)). The two `on*` tests that register a single handler and check that it fired are unaffected and run normally. diff --git a/package.json b/package.json index a2834da..851c121 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "release": "changeset publish", "test": "vitest run", "test:browser": "vitest run --config ./vitest.browser.config.ts", + "test:ios": "vitest run --config ./vitest.ios.config.ts", "test:bun": "bun run vitest run", "test:deno": "deno run -A --node-modules-dir npm:vitest run", "test:types": "npm run build && tsc -p tsconfig.types.json", diff --git a/test/helpers/iosSimulator.ts b/test/helpers/iosSimulator.ts new file mode 100644 index 0000000..2dd68e5 --- /dev/null +++ b/test/helpers/iosSimulator.ts @@ -0,0 +1,168 @@ +import {execFile} from 'node:child_process' +import {promisify} from 'node:util' + +import {defineBrowserProvider} from '@vitest/browser' +import type {BrowserProvider, TestProject} from 'vitest/node' + +const exec = promisify(execFile) + +declare module 'vitest/node' { + // `browser.instances[].browser` is typed from the union every installed provider contributes. + // `@vitest/browser-playwright` narrows it to its own three engines, so without this the iOS + // config cannot name the one browser this provider can open. + interface _BrowserNames { + iosSimulator: 'safari' + } +} + +const SAFARI_BUNDLE_ID = 'com.apple.mobilesafari' + +interface SimulatorDevice { + udid: string + name: string + state: string + isAvailable?: boolean + /** Human-readable runtime version, eg `iOS 18.6`, for the log line. */ + runtime: string + /** Runtime version as a sortable number, so the newest installed iOS is picked first. */ + sortKey: number +} + +export interface IosSimulatorOptions { + /** + * Device to run on, eg `iPhone 16`, as named by `xcrun simctl list devices`. Defaults to + * `$IOS_SIMULATOR_DEVICE`, and failing that to an iPhone on the newest installed iOS runtime - + * which runtimes and devices are installed differs between machines and between CI runner + * images, so pinning a name here means the run only works where that exact device exists. + */ + device?: string +} + +/** + * Runs the browser suite in Safari on an iOS simulator. + * + * The simulator shares the host's network stack, so the tester page reaches Vitest's Vite server + * (and the test endpoints it mounts) on the same `127.0.0.1` address the other browsers use - no + * tunnelling, and the same-origin arrangement the suite depends on still holds. + * + * Automation is `simctl` only, deliberately: nothing in the suite drives the page, it only loads + * it and reports back over the websocket Vitest already opens, so `openurl` is the entire + * integration and no WebDriver/Appium stack is needed. + */ +export function iosSimulator(options: IosSimulatorOptions = {}) { + return defineBrowserProvider({ + name: 'ios-simulator', + providerFactory: (project) => new IosSimulatorProvider(project, options), + }) +} + +class IosSimulatorProvider implements BrowserProvider { + name = 'ios-simulator' + supportsParallelism = false + + private project: TestProject + private device: string | undefined + private booted: SimulatorDevice | undefined + + constructor(project: TestProject, options: IosSimulatorOptions) { + this.project = project + this.device = options.device ?? process.env['IOS_SIMULATOR_DEVICE'] + } + + getCommandsContext(): Record { + return {} + } + + async openPage(_sessionId: string, url: string): Promise { + const device = await this.boot() + this.project.vitest.logger.log( + `Opening ${url} in Safari on ${device.name}, ${device.runtime} (${device.udid})`, + ) + await exec('xcrun', ['simctl', 'openurl', device.udid, url]) + } + + async close(): Promise { + if (!this.booted) return + // Leave the simulator booted - booting costs the better part of a minute, and a developer + // running the suite repeatedly should not pay it every time - but close Safari, so the next + // run starts on a blank page instead of restoring the previous tester page. + await exec('xcrun', ['simctl', 'terminate', this.booted.udid, SAFARI_BUNDLE_ID]).catch(() => {}) + } + + private async boot(): Promise { + if (this.booted) return this.booted + + const {stdout} = await exec('xcrun', ['simctl', 'list', 'devices', 'available', '--json']) + const devices = listDevices(stdout) + const device = pickDevice(devices, this.device) + if (!device) { + throw new Error( + `Found no available iOS simulator ${this.device ? `named "${this.device}"` : 'to run on'}. ` + + `\`xcrun simctl list devices available\` lists what is installed; ` + + `\`xcodebuild -downloadPlatform iOS\` installs a runtime if none is.`, + ) + } + + if (device.state !== 'Booted') { + // `bootstatus -b` boots the device and waits until it has finished starting up, so the + // first `openurl` does not race the boot. The simulator does not need Simulator.app to be + // open to run Safari, so nothing here brings up a UI. + await exec('xcrun', ['simctl', 'bootstatus', device.udid, '-b'], {timeout: 300_000}) + } + + this.booted = device + return device + } +} + +/** + * Flattens `simctl list devices --json` into iOS devices, newest runtime first. The JSON keys the + * device lists by runtime identifier, eg `com.apple.CoreSimulator.SimRuntime.iOS-18-6`; anything + * that is not an iOS runtime (watchOS, tvOS, visionOS) is dropped. + */ +function listDevices(json: string): SimulatorDevice[] { + const parsed: unknown = JSON.parse(json) + if (typeof parsed !== 'object' || parsed === null || !('devices' in parsed)) return [] + const {devices} = parsed + if (typeof devices !== 'object' || devices === null) return [] + + const found: SimulatorDevice[] = [] + for (const [runtime, list] of Object.entries(devices)) { + const version = /SimRuntime\.iOS-(\d+)-(\d+)/.exec(runtime) + if (!version || !Array.isArray(list)) continue + const sortKey = Number(version[1]) * 1000 + Number(version[2]) + for (const device of list) { + if (isDevice(device) && device.isAvailable !== false) { + found.push({...device, runtime: `iOS ${version[1]}.${version[2]}`, sortKey}) + } + } + } + + return found.sort((a, b) => b.sortKey - a.sortKey) +} + +function pickDevice( + devices: SimulatorDevice[], + name: string | undefined, +): SimulatorDevice | undefined { + const candidates = devices.filter((device) => + name ? device.name === name : device.name.startsWith('iPhone'), + ) + + // Prefer a device that is already booted, so a simulator the developer has open is reused + // as-is rather than a second one being started alongside it. + return candidates.find((device) => device.state === 'Booted') ?? candidates[0] +} + +function isDevice(value: unknown): value is Omit { + return ( + typeof value === 'object' && + value !== null && + 'udid' in value && + typeof value.udid === 'string' && + 'name' in value && + typeof value.name === 'string' && + 'state' in value && + typeof value.state === 'string' + ) +} diff --git a/vitest.ios.config.ts b/vitest.ios.config.ts new file mode 100644 index 0000000..869ab25 --- /dev/null +++ b/vitest.ios.config.ts @@ -0,0 +1,32 @@ +import {defineConfig} from 'vitest/config' + +import {iosSimulator} from './test/helpers/iosSimulator.ts' +import {BROWSER_PORT} from './test/helpers/routes.ts' +import {ssePlugin} from './test/helpers/ssePlugin.ts' +import {sharedConfig} from './vitest.config.ts' + +/** + * The browser suite, run in Safari on an iOS simulator - the one WebKit build we cannot get at + * through Playwright, and the one where fetch-based `EventSource` gets reported broken. + * + * macOS with Xcode's iOS platform installed only. Everything else about the arrangement matches + * `vitest.browser.config.ts`, including serving the endpoints from Vitest's own Vite server so + * the page and the endpoints stay same-origin. + */ +export default defineConfig({ + plugins: [ssePlugin()], + test: { + ...sharedConfig, + provide: {port: BROWSER_PORT, suite: 'browser'}, + browser: { + enabled: true, + provider: iosSimulator(), + // The simulator reaches the host on loopback, so this is the same address (and the same + // `localhost` second origin) the Playwright browsers use. + api: {host: '127.0.0.1', port: BROWSER_PORT}, + // Safari on a simulator cannot be run headless, and there is no browser binary to pick: + // the instance exists only to name the run. + instances: [{browser: 'safari', headless: false}], + }, + }, +}) From 0727cd06b2e018f05772bb71feba1b18b3d3e600 Mon Sep 17 00:00:00 2001 From: Espen Hovlandsdal Date: Mon, 7 Sep 2026 11:40:59 -0400 Subject: [PATCH 2/2] test: boot the iOS simulator before the suite connects Vitest's browser connect timeout is already counting down while the provider boots the simulator, so on a runner where every simulator is shut down the cold boot spends the whole budget and the run fails with "Failed to connect to the browser session within the timeout" - the provider never got as far as opening Safari. Boot up front in a step of its own, and give the connect timeout enough headroom to cover a cold boot for anyone who has not. --- .github/workflows/test.yml | 4 +++ package.json | 1 + scripts/bootIosSimulator.ts | 15 +++++++++++ test/helpers/iosSimulator.ts | 51 +++++++++++++++++++++--------------- vitest.ios.config.ts | 4 +++ 5 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 scripts/bootIosSimulator.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1acb5d4..048a323 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,5 +150,9 @@ jobs: run: npm ci - name: List available simulators run: xcrun simctl list devices available + # Separate from the test step on purpose: a cold boot takes longer than Vitest's + # `browser.connectTimeout`, which is already running while the provider boots. + - name: Boot simulator + run: npm run test:ios:boot - name: Run tests run: npm run test:ios diff --git a/package.json b/package.json index 851c121..d05aba2 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "test": "vitest run", "test:browser": "vitest run --config ./vitest.browser.config.ts", "test:ios": "vitest run --config ./vitest.ios.config.ts", + "test:ios:boot": "node scripts/bootIosSimulator.ts", "test:bun": "bun run vitest run", "test:deno": "deno run -A --node-modules-dir npm:vitest run", "test:types": "npm run build && tsc -p tsconfig.types.json", diff --git a/scripts/bootIosSimulator.ts b/scripts/bootIosSimulator.ts new file mode 100644 index 0000000..18557c4 --- /dev/null +++ b/scripts/bootIosSimulator.ts @@ -0,0 +1,15 @@ +/** + * Boots the simulator that `npm run test:ios` will use, and waits for it to finish starting up. + * + * Run as a step before the suite (see the `Test: iOS Simulator` job): a cold boot can take + * minutes, and Vitest's `browser.connectTimeout` is already counting down while the provider + * boots, so on a runner where every simulator is shut down the boot eats the whole budget and + * the run fails with "Failed to connect to the browser session within the timeout". + */ +import {bootSimulator} from '../test/helpers/iosSimulator.ts' + +const started = Date.now() +const device = await bootSimulator(process.env['IOS_SIMULATOR_DEVICE']) +const seconds = ((Date.now() - started) / 1000).toFixed(1) + +console.log(`Booted ${device.name}, ${device.runtime} (${device.udid}) in ${seconds}s`) diff --git a/test/helpers/iosSimulator.ts b/test/helpers/iosSimulator.ts index 2dd68e5..fbbce23 100644 --- a/test/helpers/iosSimulator.ts +++ b/test/helpers/iosSimulator.ts @@ -17,7 +17,7 @@ declare module 'vitest/node' { const SAFARI_BUNDLE_ID = 'com.apple.mobilesafari' -interface SimulatorDevice { +export interface SimulatorDevice { udid: string name: string state: string @@ -90,29 +90,38 @@ class IosSimulatorProvider implements BrowserProvider { } private async boot(): Promise { - if (this.booted) return this.booted - - const {stdout} = await exec('xcrun', ['simctl', 'list', 'devices', 'available', '--json']) - const devices = listDevices(stdout) - const device = pickDevice(devices, this.device) - if (!device) { - throw new Error( - `Found no available iOS simulator ${this.device ? `named "${this.device}"` : 'to run on'}. ` + - `\`xcrun simctl list devices available\` lists what is installed; ` + - `\`xcodebuild -downloadPlatform iOS\` installs a runtime if none is.`, - ) - } + this.booted ??= await bootSimulator(this.device) + return this.booted + } +} - if (device.state !== 'Booted') { - // `bootstatus -b` boots the device and waits until it has finished starting up, so the - // first `openurl` does not race the boot. The simulator does not need Simulator.app to be - // open to run Safari, so nothing here brings up a UI. - await exec('xcrun', ['simctl', 'bootstatus', device.udid, '-b'], {timeout: 300_000}) - } +/** + * Boots a simulator to run on, and resolves once it has finished starting up. + * + * Also used by `scripts/bootIosSimulator.ts`, which CI runs as a step of its own: a cold boot + * takes longer than `browser.connectTimeout` allows for, and booting up front keeps that wait + * out of the window Vitest gives the browser to connect back (and, incidentally, makes the boot + * a visible, separately timed step in the log rather than a silent stall). + */ +export async function bootSimulator(name?: string | undefined): Promise { + const {stdout} = await exec('xcrun', ['simctl', 'list', 'devices', 'available', '--json']) + const device = pickDevice(listDevices(stdout), name) + if (!device) { + throw new Error( + `Found no available iOS simulator ${name ? `named "${name}"` : 'to run on'}. ` + + `\`xcrun simctl list devices available\` lists what is installed; ` + + `\`xcodebuild -downloadPlatform iOS\` installs a runtime if none is.`, + ) + } - this.booted = device - return device + if (device.state !== 'Booted') { + // `bootstatus -b` boots the device and waits until it has finished starting up, so the first + // `openurl` does not race the boot. The simulator does not need Simulator.app to be open in + // order to run Safari, so nothing here brings up a UI. + await exec('xcrun', ['simctl', 'bootstatus', device.udid, '-b'], {timeout: 600_000}) } + + return device } /** diff --git a/vitest.ios.config.ts b/vitest.ios.config.ts index 869ab25..2fd24ca 100644 --- a/vitest.ios.config.ts +++ b/vitest.ios.config.ts @@ -24,6 +24,10 @@ export default defineConfig({ // The simulator reaches the host on loopback, so this is the same address (and the same // `localhost` second origin) the Playwright browsers use. api: {host: '127.0.0.1', port: BROWSER_PORT}, + // The default budget is spent well before a cold simulator has booted - the clock starts + // while the provider is still booting, not once the page is open. `npm run test:ios:boot` + // takes the boot out of this window; the headroom is for when it has not been run. + connectTimeout: 300_000, // Safari on a simulator cannot be run headless, and there is no browser binary to pick: // the instance exists only to name the run. instances: [{browser: 'safari', headless: false}],