Stage C67 ICU responses through Binder/Shizuku into canonical Arena

.github/workflows/ib-c67-icu-arena.yml

@@ -0,0 +1,133 @@
+name: IB C67 ICU to Arena slice
+
+on:
+  pull_request:
+    paths:
+      - 'bin/ib_android_icu_arena.grease'
+      - 'bin/ib_android_arena_read.grease'
+      - 'lib/android_binder_icu.grease'
+      - 'lib/android_shizuku_icu_provider.grease'
+      - 'native/android/ib_arena_slice.c'
+      - 'tests/test_android_icu_arena.sh'
+      - 'tests/test_android_icu_native_bridge.sh'
+      - 'tests/icu_native_transport_probe.c'
+      - '.github/workflows/ib-c67-icu-arena.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  checked-arena:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out IB
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+      - name: Check out pinned canonical appendfat arena
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          repository: isomorphisms/sd-card-append-fat
+          ref: 05a0cb7246c5a9f46abfbe92925812e7b607f454
+          path: .deps/appendfat
+          persist-credentials: false
+      - name: Check out exact ICU #35 capture source
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          repository: dilapidated-shed/icu
+          ref: 0da041fbe906657f4f705da45a842d96832e5ee1
+          path: .deps/icu
+          persist-credentials: false
+      - name: Check out independent AI-CI admitted-source verifier
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          repository: isomorphisms/ai-ci
+          ref: 634626a14834e14b776616440e0a9071708f81b7
+          path: .deps/ai-ci
+          persist-credentials: false
+      - id: ick
+        uses: isomorphisms/ai-ci/ick-host@015cc7901ae0b3ad262b476f24e129b53c56db95
+      - name: Compile storage and checked slice reader
+        env:
+          ICK: ${{ steps.ick.outputs.compiler }}
+        run: |
+          test "$(git -C .deps/appendfat rev-parse HEAD)" = 05a0cb7246c5a9f46abfbe92925812e7b607f454
+          mkdir -p build/c67-icu-arena
+          "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+            .deps/appendfat/lib/appendfat_arena.c \
+            .deps/appendfat/tools/appendfat_arena.c -o build/c67-icu-arena/appendfat_arena
+          cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+            native/android/ib_arena_slice.c -o build/c67-icu-arena/ib-arena-slice
+          sh -n bin/ib_android_icu_arena.grease
+          sh -n bin/ib_android_arena_read.grease
+          sh -n lib/android_binder_icu.grease
+          sh -n lib/android_shizuku_icu_provider.grease
+          sh -n tests/test_android_icu_arena.sh
+      - name: Run source-bound Arena and Binder refusal tests
+        env:
+          IB_APPEND_ARENA: /home/runner/work/ib/ib/build/c67-icu-arena/appendfat_arena
+          IB_ARENA_SLICE: /home/runner/work/ib/ib/build/c67-icu-arena/ib-arena-slice
+        run: |
+          export IB_APPEND_ARENA="$GITHUB_WORKSPACE/build/c67-icu-arena/appendfat_arena"
+          export IB_ARENA_SLICE="$GITHUB_WORKSPACE/build/c67-icu-arena/ib-arena-slice"
+          sh tests/test_android_icu_arena.sh
+          echo 'real_ICU_HTTPS=NOT_RUN physical_C67=NOT_RUN browser_UI=NOT_RUN'
+      - name: Compile and test actual pinned ICU #35 HTTP/TLS against IB Arena host contract
+        env:
+          ICK: ${{ steps.ick.outputs.compiler }}
+        run: |
+          set -eu
+          test "$(git -C .deps/icu rev-parse HEAD)" = 0da041fbe906657f4f705da45a842d96832e5ee1
+          sh -n tests/test_android_icu_native_bridge.sh
+          test "$(git -C .deps/ai-ci rev-parse HEAD)" = 634626a14834e14b776616440e0a9071708f81b7
+          sh -n .deps/ai-ci/ib-source/verify.sh
+          sh .deps/ai-ci/ib-source/self-test.sh > build/c67-icu-arena/aici-source-self-test.txt
+          grep -F 'PASS: independent IB v2 oracle' build/c67-icu-arena/aici-source-self-test.txt
+          test -x "$ICK"
+          "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+            -fPIC -shared .deps/icu/native/transport.c \
+            -lssl -lcrypto -o build/c67-icu-arena/libicu_transport.so
+          "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+            tests/icu_native_transport_probe.c .deps/icu/native/transport.c \
+            -lssl -lcrypto -o build/c67-icu-arena/icu-native-probe
+          export IB_APPEND_ARENA="$GITHUB_WORKSPACE/build/c67-icu-arena/appendfat_arena"
+          export IB_ARENA_SLICE="$GITHUB_WORKSPACE/build/c67-icu-arena/ib-arena-slice"
+          export IB_ICU_NATIVE_PROBE="$GITHUB_WORKSPACE/build/c67-icu-arena/icu-native-probe"
+          export IB_ICU_NATIVE_SO="$GITHUB_WORKSPACE/build/c67-icu-arena/libicu_transport.so"
+          export IB_AICI_SOURCE_VERIFY="$GITHUB_WORKSPACE/.deps/ai-ci/ib-source/verify.sh"
+          sh tests/test_android_icu_native_bridge.sh | tee build/c67-icu-arena/native-icu-bridge-receipt.txt
+          grep -F 'real compiled pinned ICU #35 native HTTP and HTTPS' \
+            build/c67-icu-arena/native-icu-bridge-receipt.txt
+          sha256sum .deps/icu/native/transport.c \
+            build/c67-icu-arena/icu-native-probe \
+            build/c67-icu-arena/libicu_transport.so \
+            > build/c67-icu-arena/native-icu-sha256.txt
+      - name: Cross-build ARM32 and ARM64 native range readers
+        run: |
+          ndk_version=27.3.13750724
+          yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" > /dev/null
+          toolchain="$ANDROID_HOME/ndk/$ndk_version/toolchains/llvm/prebuilt/linux-x86_64/bin"
+          for entry in 'armeabi-v7a armv7a-linux-androideabi24 ARM' 'arm64-v8a aarch64-linux-android24 AArch64'; do
+            set -- $entry
+            abi=$1; target=$2; machine=$3
+            mkdir -p "build/c67-icu-arena/$abi"
+            "$toolchain/$target-clang" -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+              -fPIE -pie native/android/ib_arena_slice.c \
+              -o "build/c67-icu-arena/$abi/ib-arena-slice"
+            "$toolchain/llvm-readelf" -h "build/c67-icu-arena/$abi/ib-arena-slice" \
+              > "build/c67-icu-arena/$abi/elf.txt"
+            grep -F "Machine:" "build/c67-icu-arena/$abi/elf.txt" | grep -F "$machine"
+            sha256sum "build/c67-icu-arena/$abi/ib-arena-slice" \
+              > "build/c67-icu-arena/$abi/sha256.txt"
+            printf 'abi\t%s\nphysical_device\tNOT_RUN\n' "$abi" \
+              > "build/c67-icu-arena/$abi/receipt.tsv"
+          done
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: ib-c67-arena-slice-native
+          path: |
+            build/c67-icu-arena/armeabi-v7a/
+            build/c67-icu-arena/arm64-v8a/
+            build/c67-icu-arena/native-icu-bridge-receipt.txt
+            build/c67-icu-arena/native-icu-sha256.txt
+            build/c67-icu-arena/aici-source-self-test.txt
+          if-no-files-found: error

bin/ib_android_arena_read.grease

@@ -0,0 +1,133 @@
+#!/usr/bin/env grease
+# Reopen the exact committed canonical Arena range, verify before output, and
+# check admission semantics of the versioned ICU receipt. No networking.
+set -eu
+umask 077
+export LC_ALL=C
+fail() { printf 'ib Arena read: %s\n' "$1" >&2; exit 1; }
+usage() { printf 'usage: %s REQUEST_ID PRIVATE_ARENA_DIRECTORY\n' "$0" >&2; exit 2; }
+test "$#" -eq 2 || usage
+request_id=$1
+root=$2
+case "$request_id" in ''|.|..|*[!abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-]*) usage ;; esac
+case "$root" in /*) ;; *) usage ;; esac
+test -d "$root" && test ! -L "$root" || fail 'Arena directory is absent or a symlink'
+root=$(CDPATH='' cd -P -- "$root" && pwd -P)
+case "$(stat -c '%a' "$root")" in 700|0700) ;; *) fail 'Arena root must be 0700' ;; esac
+manifest="$root/receipts/$request_id.tsv"
+arena="$root/responses.arena"
+reader=${IB_ARENA_SLICE:-}
+test -n "$reader" && test -x "$reader" || fail 'checked native reader is missing'
+test -f "$manifest" && test ! -L "$manifest" || fail 'receipt absent'
+for entry in "$arena" "$arena.used" "$arena.lock"; do
+  test -f "$entry" && test ! -L "$entry" || fail 'invalid canonical Arena file'
+done
+test "$(wc -c < "$manifest" | tr -d '[:space:]')" -le 8192 || fail 'oversized receipt'
+tr -cd '\011\012\040-\176' < "$manifest" | cmp -s - "$manifest" ||
+  fail 'receipt contains nonprintable bytes'
+field() { awk -F '\t' -v key="$1" '$1==key {print $2}' "$manifest"; }
+schema=$(field schema)
+case "$schema" in
+  ib-icu-arena-v1) count=14
+    names='schema request_id requested_url resolved_url http_status mime_type transport arena_name arena_offset arena_end byte_count sha256 reopen_check navigation_admitted' ;;
+  ib-icu-arena-v2) count=23
+    names='schema request_id source_key source_generation requested_url resolved_url http_status mime_type charset content_encoding utf8 redirect_count declared_length received_length transport arena_name arena_offset arena_end byte_count sha256 reopen_check acquisition_complete navigation_admitted' ;;
+  *) fail 'unsupported receipt schema' ;;
+esac
+awk -F '\t' -v expected="$count" -v names="$names" '
+  BEGIN {
+    count=split(names, elements, " ")
+    for (i=1; i<=count; i++) allowed[elements[i]]=1
+  }
+  NF!=2 || !allowed[$1] || $2=="" { bad=1 }
+  { seen[$1]++ }
+  END {
+    for (key in allowed) if (seen[key]!=1) bad=1
+    if (bad || NR!=expected) exit 1
+  }
+' "$manifest" || fail 'invalid receipt structure'
+test "$(field request_id)" = "$request_id" || fail 'receipt identity mismatch'
+test "$(field arena_name)" = responses.arena || fail 'wrong Arena'
+test "$(field reopen_check)" = MATCH || fail 'producer readback absent'
+test "$(field transport)" = icu-app-process-via-binder-shizuku || fail 'wrong producer'
+
+# Legacy receipts can be read as verified bytes but never authorize navigation.
+# No upgrade may infer missing HTTP observations from a v1 body.
+if test "$schema" = ib-icu-arena-v1; then
+  test "$(field navigation_admitted)" = NO || fail 'legacy navigation must be refused'
+  test "$(field resolved_url)" = UNOBSERVED || fail 'legacy resolved URL must be unobserved'
+  test "$(field http_status)" = UNOBSERVED || fail 'legacy status must be unobserved'
+  test "$(field mime_type)" = UNOBSERVED || fail 'legacy MIME must be unobserved'
+else
+  test "$(field source_key)" = "arena:$request_id" || fail 'untrusted source key'
+  source_generation=$(field source_generation)
+  case "$source_generation" in ''|0|0*|*[!0123456789]*) fail 'invalid source generation' ;; esac
+  test "${#source_generation}" -le 18 || fail 'source generation exceeds bound'
+  test "$(field acquisition_complete)" = YES || fail 'incomplete ICU acquisition'
+  status=$(field http_status)
+  case "$status" in 2[0-9][0-9]) ;; *) fail 'invalid observed HTTP status' ;; esac
+  mime=$(field mime_type)
+  case "$mime" in
+    unknown) ;;
+    */*) case "$mime" in *[!abcdefghijklmnopqrstuvwxyz0123456789.+/_-]*) fail 'invalid MIME token' ;; esac ;;
+    *) fail 'invalid MIME type' ;;
+  esac
+  charset=$(field charset)
+  case "$charset" in utf-8|UNOBSERVED) ;; *) fail 'invalid observed charset' ;; esac
+  encoding=$(field content_encoding)
+  utf8=$(field utf8)
+  case "$utf8" in valid|invalid) ;; *) fail 'invalid UTF-8 observation' ;; esac
+  requested=$(field requested_url)
+  resolved=$(field resolved_url)
+  for link in "$requested" "$resolved"; do
+    test "${#link}" -gt 8 && test "${#link}" -le 2048 || fail 'invalid URL length'
+    case "$link" in http://*|https://*) ;; *) fail 'invalid URL scheme' ;; esac
+    case "$link" in *"'"*) fail 'unsafe URL delimiter' ;; esac
+    printf '%s' "$link" | grep -Eq '^[!-&(-~]+$' || fail 'unsafe URL byte'
+  done
+  redirects=$(field redirect_count)
+  case "$redirects" in ''|*[!0123456789]*) fail 'invalid redirect count' ;; esac
+  test "${#redirects}" -le 2 && test "$redirects" -le 8 ||
+    fail 'invalid redirect observation'
+  reported=$(field received_length)
+  declared=$(field declared_length)
+  case "$reported" in ''|*[!0123456789]*) fail 'invalid received length' ;; esac
+  case "$declared" in -1) ;; ''|*[!0123456789]*) fail 'invalid declared length' ;; esac
+  admission=$(field navigation_admitted)
+  case "$admission" in YES|NO) ;; *) fail 'invalid admission state' ;; esac
+  if test "$admission" = YES; then
+    test "$status" = 200 && test "$mime" = text/html &&
+      test "$charset" = utf-8 && test "$encoding" = identity &&
+      test "$utf8" = valid || fail 'forged navigation admission'
+  fi
+fi
+
+begin=$(field arena_offset)
+end=$(field arena_end)
+length=$(field byte_count)
+expected=$(field sha256)
+for number in "$begin" "$end" "$length"; do
+  case "$number" in ''|*[!0123456789]*) fail 'invalid source range' ;; esac
+  test "${#number}" -le 18 || fail 'source range exceeds implemented bound'
+done
+test "$length" -gt 0 && test "$length" -le 4194304 || fail 'invalid source length'
+test "$((begin + length))" -eq "$end" || fail 'receipt range mismatch'
+test "${#expected}" -eq 64 || fail 'bad digest length'
+case "$expected" in *[!0123456789abcdef]*) fail 'bad digest characters' ;; esac
+if test "$schema" = ib-icu-arena-v2; then
+  test "$reported" -eq "$length" || fail 'observed received length mismatch'
+  if test "$declared" != -1; then
+    test "$declared" -eq "$length" || fail 'declared length mismatch'
+  fi
+fi
+
+test -d "$root/staging" && test ! -L "$root/staging" || fail 'staging absent'
+work=$(mktemp -d "$root/staging/reader.$request_id.XXXXXXXX") || fail 'private read stage failed'
+cleanup() { rm -rf "$work"; }
+trap cleanup EXIT HUP INT TERM
+"$reader" slice "$arena" "$begin" "$length" > "$work/body" || fail 'committed range unreadable'
+test "$(wc -c < "$work/body" | tr -d '[:space:]')" -eq "$length" || fail 'short source range'
+observed=$(sha256sum "$work/body" | cut -d ' ' -f 1)
+test "$observed" = "$expected" || fail 'source digest mismatch'
+# Never emit even a prefix before the entire committed range is validated.
+cat "$work/body"