diff --git a/.github/workflows/ib-c67-icu-arena.yml b/.github/workflows/ib-c67-icu-arena.yml new file mode 100644 index 00000000..3bb62aa2 --- /dev/null +++ b/.github/workflows/ib-c67-icu-arena.yml @@ -0,0 +1,133 @@ +name: IB C67 ICU to Arena slice + +on: + pull_request: + paths: + - 'bin/ib_android_icu_arena.grease' + - 'bin/ib_android_arena_read.grease' + - 'lib/android_binder_icu.grease' + - 'lib/android_shizuku_icu_provider.grease' + - 'native/android/ib_arena_slice.c' + - 'tests/test_android_icu_arena.sh' + - 'tests/test_android_icu_native_bridge.sh' + - 'tests/icu_native_transport_probe.c' + - '.github/workflows/ib-c67-icu-arena.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + checked-arena: + runs-on: ubuntu-24.04 + steps: + - name: Check out IB + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - name: Check out pinned canonical appendfat arena + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: isomorphisms/sd-card-append-fat + ref: 05a0cb7246c5a9f46abfbe92925812e7b607f454 + path: .deps/appendfat + persist-credentials: false + - name: Check out exact ICU #35 capture source + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: dilapidated-shed/icu + ref: 0da041fbe906657f4f705da45a842d96832e5ee1 + path: .deps/icu + persist-credentials: false + - name: Check out independent AI-CI admitted-source verifier + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: isomorphisms/ai-ci + ref: 634626a14834e14b776616440e0a9071708f81b7 + path: .deps/ai-ci + persist-credentials: false + - id: ick + uses: isomorphisms/ai-ci/ick-host@015cc7901ae0b3ad262b476f24e129b53c56db95 + - name: Compile storage and checked slice reader + env: + ICK: ${{ steps.ick.outputs.compiler }} + run: | + test "$(git -C .deps/appendfat rev-parse HEAD)" = 05a0cb7246c5a9f46abfbe92925812e7b607f454 + mkdir -p build/c67-icu-arena + "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + .deps/appendfat/lib/appendfat_arena.c \ + .deps/appendfat/tools/appendfat_arena.c -o build/c67-icu-arena/appendfat_arena + cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + native/android/ib_arena_slice.c -o build/c67-icu-arena/ib-arena-slice + sh -n bin/ib_android_icu_arena.grease + sh -n bin/ib_android_arena_read.grease + sh -n lib/android_binder_icu.grease + sh -n lib/android_shizuku_icu_provider.grease + sh -n tests/test_android_icu_arena.sh + - name: Run source-bound Arena and Binder refusal tests + env: + IB_APPEND_ARENA: /home/runner/work/ib/ib/build/c67-icu-arena/appendfat_arena + IB_ARENA_SLICE: /home/runner/work/ib/ib/build/c67-icu-arena/ib-arena-slice + run: | + export IB_APPEND_ARENA="$GITHUB_WORKSPACE/build/c67-icu-arena/appendfat_arena" + export IB_ARENA_SLICE="$GITHUB_WORKSPACE/build/c67-icu-arena/ib-arena-slice" + sh tests/test_android_icu_arena.sh + echo 'real_ICU_HTTPS=NOT_RUN physical_C67=NOT_RUN browser_UI=NOT_RUN' + - name: Compile and test actual pinned ICU #35 HTTP/TLS against IB Arena host contract + env: + ICK: ${{ steps.ick.outputs.compiler }} + run: | + set -eu + test "$(git -C .deps/icu rev-parse HEAD)" = 0da041fbe906657f4f705da45a842d96832e5ee1 + sh -n tests/test_android_icu_native_bridge.sh + test "$(git -C .deps/ai-ci rev-parse HEAD)" = 634626a14834e14b776616440e0a9071708f81b7 + sh -n .deps/ai-ci/ib-source/verify.sh + sh .deps/ai-ci/ib-source/self-test.sh > build/c67-icu-arena/aici-source-self-test.txt + grep -F 'PASS: independent IB v2 oracle' build/c67-icu-arena/aici-source-self-test.txt + test -x "$ICK" + "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + -fPIC -shared .deps/icu/native/transport.c \ + -lssl -lcrypto -o build/c67-icu-arena/libicu_transport.so + "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + tests/icu_native_transport_probe.c .deps/icu/native/transport.c \ + -lssl -lcrypto -o build/c67-icu-arena/icu-native-probe + export IB_APPEND_ARENA="$GITHUB_WORKSPACE/build/c67-icu-arena/appendfat_arena" + export IB_ARENA_SLICE="$GITHUB_WORKSPACE/build/c67-icu-arena/ib-arena-slice" + export IB_ICU_NATIVE_PROBE="$GITHUB_WORKSPACE/build/c67-icu-arena/icu-native-probe" + export IB_ICU_NATIVE_SO="$GITHUB_WORKSPACE/build/c67-icu-arena/libicu_transport.so" + export IB_AICI_SOURCE_VERIFY="$GITHUB_WORKSPACE/.deps/ai-ci/ib-source/verify.sh" + sh tests/test_android_icu_native_bridge.sh | tee build/c67-icu-arena/native-icu-bridge-receipt.txt + grep -F 'real compiled pinned ICU #35 native HTTP and HTTPS' \ + build/c67-icu-arena/native-icu-bridge-receipt.txt + sha256sum .deps/icu/native/transport.c \ + build/c67-icu-arena/icu-native-probe \ + build/c67-icu-arena/libicu_transport.so \ + > build/c67-icu-arena/native-icu-sha256.txt + - name: Cross-build ARM32 and ARM64 native range readers + run: | + ndk_version=27.3.13750724 + yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" > /dev/null + toolchain="$ANDROID_HOME/ndk/$ndk_version/toolchains/llvm/prebuilt/linux-x86_64/bin" + for entry in 'armeabi-v7a armv7a-linux-androideabi24 ARM' 'arm64-v8a aarch64-linux-android24 AArch64'; do + set -- $entry + abi=$1; target=$2; machine=$3 + mkdir -p "build/c67-icu-arena/$abi" + "$toolchain/$target-clang" -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + -fPIE -pie native/android/ib_arena_slice.c \ + -o "build/c67-icu-arena/$abi/ib-arena-slice" + "$toolchain/llvm-readelf" -h "build/c67-icu-arena/$abi/ib-arena-slice" \ + > "build/c67-icu-arena/$abi/elf.txt" + grep -F "Machine:" "build/c67-icu-arena/$abi/elf.txt" | grep -F "$machine" + sha256sum "build/c67-icu-arena/$abi/ib-arena-slice" \ + > "build/c67-icu-arena/$abi/sha256.txt" + printf 'abi\t%s\nphysical_device\tNOT_RUN\n' "$abi" \ + > "build/c67-icu-arena/$abi/receipt.tsv" + done + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ib-c67-arena-slice-native + path: | + build/c67-icu-arena/armeabi-v7a/ + build/c67-icu-arena/arm64-v8a/ + build/c67-icu-arena/native-icu-bridge-receipt.txt + build/c67-icu-arena/native-icu-sha256.txt + build/c67-icu-arena/aici-source-self-test.txt + if-no-files-found: error diff --git a/bin/ib_android_arena_read.grease b/bin/ib_android_arena_read.grease new file mode 100644 index 00000000..20c387b7 --- /dev/null +++ b/bin/ib_android_arena_read.grease @@ -0,0 +1,133 @@ +#!/usr/bin/env grease +# Reopen the exact committed canonical Arena range, verify before output, and +# check admission semantics of the versioned ICU receipt. No networking. +set -eu +umask 077 +export LC_ALL=C +fail() { printf 'ib Arena read: %s\n' "$1" >&2; exit 1; } +usage() { printf 'usage: %s REQUEST_ID PRIVATE_ARENA_DIRECTORY\n' "$0" >&2; exit 2; } +test "$#" -eq 2 || usage +request_id=$1 +root=$2 +case "$request_id" in ''|.|..|*[!abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-]*) usage ;; esac +case "$root" in /*) ;; *) usage ;; esac +test -d "$root" && test ! -L "$root" || fail 'Arena directory is absent or a symlink' +root=$(CDPATH='' cd -P -- "$root" && pwd -P) +case "$(stat -c '%a' "$root")" in 700|0700) ;; *) fail 'Arena root must be 0700' ;; esac +manifest="$root/receipts/$request_id.tsv" +arena="$root/responses.arena" +reader=${IB_ARENA_SLICE:-} +test -n "$reader" && test -x "$reader" || fail 'checked native reader is missing' +test -f "$manifest" && test ! -L "$manifest" || fail 'receipt absent' +for entry in "$arena" "$arena.used" "$arena.lock"; do + test -f "$entry" && test ! -L "$entry" || fail 'invalid canonical Arena file' +done +test "$(wc -c < "$manifest" | tr -d '[:space:]')" -le 8192 || fail 'oversized receipt' +tr -cd '\011\012\040-\176' < "$manifest" | cmp -s - "$manifest" || + fail 'receipt contains nonprintable bytes' +field() { awk -F '\t' -v key="$1" '$1==key {print $2}' "$manifest"; } +schema=$(field schema) +case "$schema" in + ib-icu-arena-v1) count=14 + names='schema request_id requested_url resolved_url http_status mime_type transport arena_name arena_offset arena_end byte_count sha256 reopen_check navigation_admitted' ;; + ib-icu-arena-v2) count=23 + names='schema request_id source_key source_generation requested_url resolved_url http_status mime_type charset content_encoding utf8 redirect_count declared_length received_length transport arena_name arena_offset arena_end byte_count sha256 reopen_check acquisition_complete navigation_admitted' ;; + *) fail 'unsupported receipt schema' ;; +esac +awk -F '\t' -v expected="$count" -v names="$names" ' + BEGIN { + count=split(names, elements, " ") + for (i=1; i<=count; i++) allowed[elements[i]]=1 + } + NF!=2 || !allowed[$1] || $2=="" { bad=1 } + { seen[$1]++ } + END { + for (key in allowed) if (seen[key]!=1) bad=1 + if (bad || NR!=expected) exit 1 + } +' "$manifest" || fail 'invalid receipt structure' +test "$(field request_id)" = "$request_id" || fail 'receipt identity mismatch' +test "$(field arena_name)" = responses.arena || fail 'wrong Arena' +test "$(field reopen_check)" = MATCH || fail 'producer readback absent' +test "$(field transport)" = icu-app-process-via-binder-shizuku || fail 'wrong producer' + +# Legacy receipts can be read as verified bytes but never authorize navigation. +# No upgrade may infer missing HTTP observations from a v1 body. +if test "$schema" = ib-icu-arena-v1; then + test "$(field navigation_admitted)" = NO || fail 'legacy navigation must be refused' + test "$(field resolved_url)" = UNOBSERVED || fail 'legacy resolved URL must be unobserved' + test "$(field http_status)" = UNOBSERVED || fail 'legacy status must be unobserved' + test "$(field mime_type)" = UNOBSERVED || fail 'legacy MIME must be unobserved' +else + test "$(field source_key)" = "arena:$request_id" || fail 'untrusted source key' + source_generation=$(field source_generation) + case "$source_generation" in ''|0|0*|*[!0123456789]*) fail 'invalid source generation' ;; esac + test "${#source_generation}" -le 18 || fail 'source generation exceeds bound' + test "$(field acquisition_complete)" = YES || fail 'incomplete ICU acquisition' + status=$(field http_status) + case "$status" in 2[0-9][0-9]) ;; *) fail 'invalid observed HTTP status' ;; esac + mime=$(field mime_type) + case "$mime" in + unknown) ;; + */*) case "$mime" in *[!abcdefghijklmnopqrstuvwxyz0123456789.+/_-]*) fail 'invalid MIME token' ;; esac ;; + *) fail 'invalid MIME type' ;; + esac + charset=$(field charset) + case "$charset" in utf-8|UNOBSERVED) ;; *) fail 'invalid observed charset' ;; esac + encoding=$(field content_encoding) + utf8=$(field utf8) + case "$utf8" in valid|invalid) ;; *) fail 'invalid UTF-8 observation' ;; esac + requested=$(field requested_url) + resolved=$(field resolved_url) + for link in "$requested" "$resolved"; do + test "${#link}" -gt 8 && test "${#link}" -le 2048 || fail 'invalid URL length' + case "$link" in http://*|https://*) ;; *) fail 'invalid URL scheme' ;; esac + case "$link" in *"'"*) fail 'unsafe URL delimiter' ;; esac + printf '%s' "$link" | grep -Eq '^[!-&(-~]+$' || fail 'unsafe URL byte' + done + redirects=$(field redirect_count) + case "$redirects" in ''|*[!0123456789]*) fail 'invalid redirect count' ;; esac + test "${#redirects}" -le 2 && test "$redirects" -le 8 || + fail 'invalid redirect observation' + reported=$(field received_length) + declared=$(field declared_length) + case "$reported" in ''|*[!0123456789]*) fail 'invalid received length' ;; esac + case "$declared" in -1) ;; ''|*[!0123456789]*) fail 'invalid declared length' ;; esac + admission=$(field navigation_admitted) + case "$admission" in YES|NO) ;; *) fail 'invalid admission state' ;; esac + if test "$admission" = YES; then + test "$status" = 200 && test "$mime" = text/html && + test "$charset" = utf-8 && test "$encoding" = identity && + test "$utf8" = valid || fail 'forged navigation admission' + fi +fi + +begin=$(field arena_offset) +end=$(field arena_end) +length=$(field byte_count) +expected=$(field sha256) +for number in "$begin" "$end" "$length"; do + case "$number" in ''|*[!0123456789]*) fail 'invalid source range' ;; esac + test "${#number}" -le 18 || fail 'source range exceeds implemented bound' +done +test "$length" -gt 0 && test "$length" -le 4194304 || fail 'invalid source length' +test "$((begin + length))" -eq "$end" || fail 'receipt range mismatch' +test "${#expected}" -eq 64 || fail 'bad digest length' +case "$expected" in *[!0123456789abcdef]*) fail 'bad digest characters' ;; esac +if test "$schema" = ib-icu-arena-v2; then + test "$reported" -eq "$length" || fail 'observed received length mismatch' + if test "$declared" != -1; then + test "$declared" -eq "$length" || fail 'declared length mismatch' + fi +fi + +test -d "$root/staging" && test ! -L "$root/staging" || fail 'staging absent' +work=$(mktemp -d "$root/staging/reader.$request_id.XXXXXXXX") || fail 'private read stage failed' +cleanup() { rm -rf "$work"; } +trap cleanup EXIT HUP INT TERM +"$reader" slice "$arena" "$begin" "$length" > "$work/body" || fail 'committed range unreadable' +test "$(wc -c < "$work/body" | tr -d '[:space:]')" -eq "$length" || fail 'short source range' +observed=$(sha256sum "$work/body" | cut -d ' ' -f 1) +test "$observed" = "$expected" || fail 'source digest mismatch' +# Never emit even a prefix before the entire committed range is validated. +cat "$work/body" diff --git a/bin/ib_android_icu_arena.grease b/bin/ib_android_icu_arena.grease new file mode 100644 index 00000000..7be67465 --- /dev/null +++ b/bin/ib_android_icu_arena.grease @@ -0,0 +1,178 @@ +#!/usr/bin/env grease +# ICU #35 observed response metadata and exact body -> canonical userspace +# appendfat Arena. A receipt is published only after ICU exits successfully, +# metadata passes structural checks, and committed bytes are reopened. +set -eu +umask 077 +export LC_ALL=C + +case "$0" in */*) script_directory=${0%/*} ;; *) script_directory=. ;; esac +repository_root=$(CDPATH='' cd -- "$script_directory/.." && pwd) +. "$repository_root/lib/android_shizuku_host.grease" +. "$repository_root/lib/android_shizuku_icu_provider.grease" +. "$repository_root/lib/android_binder_icu.grease" + +fail() { printf 'ib ICU/Arena: %s\n' "$1" >&2; exit 1; } +usage() { printf 'usage: %s REQUEST_ID HTTP_URL PRIVATE_ARENA_DIRECTORY SOURCE_GENERATION\n' "$0" >&2; exit 2; } + +test "$#" -eq 4 || usage +request_id=$1 +url=$2 +root=$3 +source_generation=$4 +case "$request_id" in ''|.|..|*[!abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-]*) usage ;; esac +ib_shizuku_icu_url "$url" || usage +case "$root" in /*) ;; *) usage ;; esac +case "$source_generation" in ''|0|0*|*[!0123456789]*) usage ;; esac +test "${#source_generation}" -le 18 || usage +test -d "$root" && test ! -L "$root" || fail 'private Arena directory absent or a symlink' +root=$(CDPATH='' cd -P -- "$root" && pwd -P) +case "$(stat -c '%a' "$root")" in 700|0700) ;; *) fail 'Arena root must be owner-only mode 0700' ;; esac +for directory in "$root/staging" "$root/receipts" "$root/locks"; do + test ! -L "$directory" || fail 'symlink below Arena root' + mkdir -p "$directory" + test -d "$directory" && test ! -L "$directory" || fail 'bad Arena subdirectory' +done +arena=$root/responses.arena +for path in "$arena" "$arena.used" "$arena.lock"; do + test -f "$path" && test ! -L "$path" || fail 'Arena is not preinitialized by its owner' +done +arena_tool=${IB_APPEND_ARENA:-} +slice_tool=${IB_ARENA_SLICE:-} +test -n "$arena_tool" && test -x "$arena_tool" || fail 'canonical appendfat_arena executable required' +test -n "$slice_tool" && test -x "$slice_tool" || fail 'checked ib-arena-slice executable required' + +receipt=$root/receipts/$request_id.tsv +test ! -e "$receipt" && test ! -L "$receipt" || fail 'request identity already published' +lock=$root/locks/$request_id +mkdir "$lock" || fail 'request identity is already running' +# Lock survives failure; a failed request ID may not be silently replayed. +work=$(mktemp -d "$root/staging/$request_id.XXXXXXXX") || fail 'cannot create private staging' +cleanup() { rm -rf "$work"; } +trap cleanup EXIT HUP INT TERM +capture=$work/icu.body +metadata=$work/icu.metadata + +authority=$(ib_shizuku_authority) || fail 'Shizuku unavailable or unauthorized' +case "$authority" in shell|root) ;; *) fail 'Shizuku lacks authorized shell/root authority' ;; esac +if ! ib_binder_icu_get "$url" "$capture" "$metadata"; then + fail 'ICU capture or Shizuku failed; no Arena bytes were committed' +fi +for file in "$capture" "$metadata"; do + test -f "$file" && test ! -L "$file" || fail 'missing or linked ICU capture' +done +bytes=$(wc -c < "$capture" | tr -d '[:space:]') +case "$bytes" in ''|*[!0123456789]*) fail 'bad ICU body size' ;; esac +test "$bytes" -gt 0 && test "$bytes" -le 4194304 || fail 'ICU body empty or exceeds 4 MiB' +metadata_bytes=$(wc -c < "$metadata" | tr -d '[:space:]') +test "$metadata_bytes" -gt 0 && test "$metadata_bytes" -le 8192 || fail 'bad ICU metadata length' + +# PR #35's metadata has exactly eight named fields. Reject injected lines, +# duplicate/missing keys, control bytes, CR and arbitrary extension fields. +tr -cd '\011\012\040-\176' < "$metadata" | cmp -s - "$metadata" || + fail 'ICU metadata includes non-printable bytes' +awk -F '\t' ' + BEGIN { + count=split("http_status content_type content_encoding declared_length received_length utf8 resolved_url redirect_count", names, " ") + for (i=1;i<=count;i++) allowed[names[i]]=1 + } + NF!=2 || !allowed[$1] || $2=="" { bad=1 } + { seen[$1]++ } + END { + for (key in allowed) if (seen[key]!=1) bad=1 + if (bad || NR!=8) exit 1 + } +' "$metadata" || fail 'invalid ICU metadata structure' +field() { awk -F '\t' -v name="$1" '$1==name {print $2}' "$metadata"; } + +http_status=$(field http_status) +content_type=$(field content_type) +content_encoding=$(field content_encoding) +declared_length=$(field declared_length) +received_length=$(field received_length) +utf8=$(field utf8) +resolved_url=$(field resolved_url) +redirect_count=$(field redirect_count) + +# ICU #35 returns success only for complete 2xx responses; check its observed +# fields independently before making any body bytes reader-eligible. +case "$http_status" in 2[0-9][0-9]) ;; *) fail 'unobserved or unsuccessful HTTP status' ;; esac +ib_shizuku_icu_url "$resolved_url" || fail 'unsupported or unsafe resolved URL' +case "$received_length" in ''|*[!0123456789]*) fail 'bad observed received length' ;; esac +test "$received_length" -eq "$bytes" || fail 'ICU byte count disagrees with capture' +case "$declared_length" in + -1) ;; + ''|*[!0123456789]*) fail 'invalid declared HTTP length' ;; + *) test "$declared_length" -eq "$bytes" || fail 'incomplete HTTP body' ;; +esac +case "$redirect_count" in ''|*[!0123456789]*) fail 'invalid redirect count' ;; esac +test "${#redirect_count}" -le 2 && test "$redirect_count" -le 8 || + fail 'redirect count exceeds ICU bound' +case "$utf8" in valid|invalid) ;; *) fail 'invalid ICU UTF-8 observation' ;; esac + +# Preserve MIME and explicit charset separately. Byte-level UTF-8 validity is +# not a substitute for an observed/declared charset. +lower_type=$(printf '%s' "$content_type" | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz') +mime_type=$(printf '%s' "${lower_type%%;*}" | tr -d ' ') +case "$mime_type" in + unknown) ;; + */*) case "$mime_type" in *[!abcdefghijklmnopqrstuvwxyz0123456789.+/_-]*) fail 'invalid MIME token' ;; esac ;; + *) fail 'invalid MIME type' ;; +esac +charset=UNOBSERVED +case "$lower_type" in + *';'*) + parameters=$(printf '%s' "${lower_type#*;}" | tr -d ' ') + case "$parameters" in charset=utf-8|charset=\"utf-8\") charset=utf-8 ;; esac + ;; +esac +navigation_admitted=NO +if test "$http_status" = 200 && test "$mime_type" = text/html && + test "$charset" = utf-8 && test "$content_encoding" = identity && + test "$utf8" = valid; then + navigation_admitted=YES +fi +digest=$(sha256sum "$capture" | cut -d ' ' -f 1) + +# Canonical appendfat_arena owns locking, fsync and the commit pointer. Never +# publish metadata based only on an append acknowledgment. +append_result=$("$arena_tool" append "$arena" "$capture") || fail 'Arena append failed' +old=$(printf '%s\n' "$append_result" | sed -n 's/^APPENDED old_used=\([0-9][0-9]*\) new_used=[0-9][0-9]* capacity_bytes=[0-9][0-9]*$/\1/p') +new=$(printf '%s\n' "$append_result" | sed -n 's/^APPENDED old_used=[0-9][0-9]* new_used=\([0-9][0-9]*\) capacity_bytes=[0-9][0-9]*$/\1/p') +test -n "$old" && test -n "$new" || fail 'unrecognized Arena commit receipt' +test "$new" -ge "$old" && test "$((new - old))" -eq "$bytes" || + fail 'Arena appended different byte count' +"$slice_tool" slice "$arena" "$old" "$bytes" > "$work/reopened" || + fail 'Arena committed range unreadable' +cmp -s "$capture" "$work/reopened" || fail 'Arena reopened bytes mismatch' + +{ + printf 'schema\tib-icu-arena-v2\n' + printf 'request_id\t%s\n' "$request_id" + printf 'source_key\tarena:%s\n' "$request_id" + printf 'source_generation\t%s\n' "$source_generation" + printf 'requested_url\t%s\n' "$url" + printf 'resolved_url\t%s\n' "$resolved_url" + printf 'http_status\t%s\n' "$http_status" + printf 'mime_type\t%s\n' "$mime_type" + printf 'charset\t%s\n' "$charset" + printf 'content_encoding\t%s\n' "$content_encoding" + printf 'utf8\t%s\n' "$utf8" + printf 'redirect_count\t%s\n' "$redirect_count" + printf 'declared_length\t%s\n' "$declared_length" + printf 'received_length\t%s\n' "$received_length" + printf 'transport\ticu-app-process-via-binder-shizuku\n' + printf 'arena_name\tresponses.arena\n' + printf 'arena_offset\t%s\n' "$old" + printf 'arena_end\t%s\n' "$new" + printf 'byte_count\t%s\n' "$bytes" + printf 'sha256\t%s\n' "$digest" + printf 'reopen_check\tMATCH\n' + printf 'acquisition_complete\tYES\n' + printf 'navigation_admitted\t%s\n' "$navigation_admitted" +} > "$work/receipt.tsv" +"$slice_tool" sync "$work/receipt.tsv" || fail 'could not sync staged receipt' +test ! -e "$receipt" || fail 'receipt already exists before publication' +mv "$work/receipt.tsv" "$receipt" || fail 'cannot publish immutable receipt' +"$slice_tool" sync "$receipt" || fail 'could not sync published receipt' +printf 'committed\t%s\t%s\t%s\t%s\n' "$request_id" "$old" "$bytes" "$navigation_admitted" diff --git a/docs/c67-icu-arena.md b/docs/c67-icu-arena.md new file mode 100644 index 00000000..291becc0 --- /dev/null +++ b/docs/c67-icu-arena.md @@ -0,0 +1,112 @@ +# C67 ICU #35 → Shizuku → Arena integration candidate + +Status: **host-contract implementation; physical ICU/Shizuku/reader acceptance NOT_RUN**. +This is not a C67 URL-capable release or a claim that ICU #35 is merged. + +## Acquisition and provenance + +The pinned candidate API is [dilapidated-shed/icu PR #35](https://github.com/dilapidated-shed/icu/pull/35), +head commit 0da041fbe906657f4f705da45a842d96832e5ee1 at integration. +Do **not** substitute older ICU DEX/native objects: old versions write only +body stdout, not the paired ICU_CAPTURE_BODY_PATH and +ICU_CAPTURE_METADATA_PATH files required here. + + browser-owned request (future scoped Binder/PFD handoff) + → IB Binder-shaped ICU capability (current Shizuku rish lowering) + → verify deployed ICU DEX + native TLS .so against explicit SHA-256 pins + → one Idric.Generated get via Android app_process + → ICU #35 creates private paired body + metadata under the shell UID + → single bounded, length-framed rish stdout transfer + → independently validate metadata shape and captured byte count + → canonical preinitialized appendfat userspace Arena commit + → native C reopen + compare exact range with staged captured body + → immutable v2 Arena receipt; admit only qualified HTML + → Idriç HTML extraction/prepaint in IB PR #154 [NOT WIRED] + → native Android reader [NOT WIRED] + +This stays in the existing userspace sd-card-append-fat Arena, not a new +browser store, app-side HTTP fetch, WebView transport, or Java service. + +### Setup and development invocation + +The owner supplies a non-symlink private root (mode 0700), creates the +canonical responses.arena and selects checked executable paths: + + appendfat_arena create PRIVATE_DIR/responses.arena 16777216 + +Set IB_APPEND_ARENA and IB_ARENA_SLICE to the pinned compiled executables. +Set IB_C67_ICU_DEX, IB_C67_ICU_SO, IB_C67_ICU_DEX_SHA256, +IB_C67_ICU_SO_SHA256 to **actual qualified** Android app_process DEX and +ARM native ICU #35 artifacts in /data/local/tmp/ib-icu/. This PR does not +produce these artifacts or invent their SHA-256 values. Shizuku must be +started, and the caller must be authorized as shell/root. + + sh bin/ib_android_icu_arena.grease REQUEST_ID https://example.com/ PRIVATE_DIR SOURCE_GENERATION + +SOURCE_GENERATION is a nonzero canonical decimal integer (max 18 digits) +allocated by the browser/source owner, **not inferred from HTTP**. The +caller remains responsible for ordering generations and binding user intent; +this development command is not an app IPC contract. + +The Shizuku UID cannot write directly into the application's private Arena. +Its one trusted shell invocation makes a random mode-0700 staging directory +inside the pinned /data/local/tmp/ib-icu/, passes both ICU capture paths +through environment variables, then transports a single byte-counted packet: + + IBICU01 METADATA_BYTES BODY_BYTES\n + [exact eight-field ICU metadata] + [exact binary response body] + +The packet header contains two eight-digit decimal byte lengths and a +newline. Metadata is limited to 8192 bytes; the body to 4 MiB, stricter +than ICU's 8 MiB native limit. The untrusted packet and staging files are +discarded on failures. The reader never sees a partial packet. + +## Receipt versions and navigation policy + +ib-icu-arena-v1 is still safely readable as verified **body-only** data +with UNOBSERVED status, MIME and final URL. It **cannot** be upgraded into +navigation by guessing those fields. + +ib-icu-arena-v2 records exactly 23 named fields including: + +- Request identity, source_key=arena:REQUEST_ID, source_generation. +- Requested URL and **ICU-observed** final resolved_url plus redirect_count. +- HTTP status, MIME type, explicit charset, content_encoding, ICU-reported + utf8, declared_length and received_length. +- Canonical arena_offset, arena_end, byte_count and sha256. +- reopen_check=MATCH, acquisition_complete=YES, navigation_admitted. + +The producer checks the exact eight-field upstream ICU metadata schema, +received byte count, declared Content-Length (if present), redirect limit, +URL syntax, nonzero 2xx status, and independently reopened Arena range. + +navigation_admitted=YES currently requires **all** of: HTTP 200, +text/html, explicitly declared charset=utf-8, identity content +encoding, valid ICU UTF-8 observation, complete acquisition, intact +reopened bytes, source key/generation and correct SHA-256. Unsupported +text encodings, unknown MIME, missing charset, compressed undecoded bodies +and non-HTML responses remain NO; they are not converted to prepaint. +The reader independently rechecks the v2 admission predicate before +emitting even a byte. The external IB #154 adapter should still check +the receipt and call the verified reader, not bypass it. + +The receipt is published only after append, native reopen and body compare. +Failure after append but before receipt leaves only an unreferenced tail; +no browser history is updated. Each request ID is single-use with a retained +per-request lock even on failure. + +## Evidence and unfinished qualification + +The workflow uses the **real pinned canonical appendfat_arena** and native +range reader. It mocks exactly the Shizuku command's framed stdout, +including redirect metadata, verifies v1/v2 reading, original binary bytes, +and deliberate missing/duplicate/forged metadata, corrupt digest and +commit-pointer refusals. ICU #35 has its own **separate** real native +redirect integration tests; passing IB host mocks is not passing that test. + +Still **NOT_RUN** here: pinned ICU #35 Android DEX/ARM32 build and installed +artifact digests, actual TLS HTTPS and Shizuku invocation on C67, cross-UID +scoped Binder/PFD into the app, end-to-end #154 Idriç conversion, Material +reader URL entry and clicked links, C67 durable history/restart. +Coordinate IB #151, #142, #153, #154, #129 and ICU #35 before merge/release. diff --git a/lib/android_binder_icu.grease b/lib/android_binder_icu.grease new file mode 100644 index 00000000..bb1f819a --- /dev/null +++ b/lib/android_binder_icu.grease @@ -0,0 +1,5 @@ +# Selected Binder capability: ICU acquisition. Shizuku currently supplies this +# capability, but a browser/reader never invokes rish or opens network sockets. +ib_binder_icu_get() { + ib_shizuku_icu_get "$@" +} diff --git a/lib/android_shizuku_icu_provider.grease b/lib/android_shizuku_icu_provider.grease new file mode 100644 index 00000000..e3292c84 --- /dev/null +++ b/lib/android_shizuku_icu_provider.grease @@ -0,0 +1,129 @@ +# Shizuku-specific ICU Android app_process lowering. +# PR dilapidated-shed/icu#35: one checked request captures binary body and +# observed HTTP metadata together. Both originate under the shell UID; this +# adapter transports one bounded, explicitly framed stream back to the owner. +# IB never treats unverified rish stdout as a page or metadata as a guess. + +ib_shizuku_icu_atom_path() { + case "$1" in + /data/local/tmp/ib-icu/*) ;; + *) return 2 ;; + esac + case "$1" in + *..*|*[!abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._/-]*) return 2 ;; + esac +} + +ib_shizuku_icu_sha256() { + test "${#1}" -eq 64 || return 2 + case "$1" in ''|*[!0123456789abcdef]*) return 2 ;; esac +} + +ib_shizuku_icu_url() { + test "${#1}" -gt 8 && test "${#1}" -le 2048 || return 2 + case "$1" in http://*|https://*) ;; *) return 2 ;; esac + case "$1" in *"'"*) return 2 ;; esac + printf '%s' "$1" | LC_ALL=C grep -Eq '^[!-&(-~]+$' +} + +# The frame is "IBICU01 %08d %08d\n" followed by exact metadata bytes and +# exact body bytes. A single rish invocation owns ICU's temporary output files. +# No consumer is allowed to parse raw body bytes as lines or infer HTTP fields. +ib_shizuku_icu_get() { + if test "$#" -ne 3; then + printf 'ib_shizuku_icu_get requires URL BODY_OUTPUT METADATA_OUTPUT\n' >&2 + return 2 + fi + icu_url=$1 + icu_output=$2 + icu_metadata=$3 + icu_dex=${IB_C67_ICU_DEX:-} + icu_so=${IB_C67_ICU_SO:-} + icu_dex_sha=${IB_C67_ICU_DEX_SHA256:-} + icu_so_sha=${IB_C67_ICU_SO_SHA256:-} + ib_shizuku_icu_url "$icu_url" || { printf 'unsupported or unsafe ICU URL\n' >&2; return 2; } + ib_shizuku_icu_atom_path "$icu_dex" || { printf 'unapproved ICU DEX path\n' >&2; return 2; } + ib_shizuku_icu_atom_path "$icu_so" || { printf 'unapproved ICU native path\n' >&2; return 2; } + ib_shizuku_icu_sha256 "$icu_dex_sha" || { printf 'missing pinned ICU DEX hash\n' >&2; return 2; } + ib_shizuku_icu_sha256 "$icu_so_sha" || { printf 'missing pinned ICU native hash\n' >&2; return 2; } + test "$icu_output" != "$icu_metadata" || return 2 + test ! -e "$icu_output" && test ! -L "$icu_output" || { printf 'reused ICU body path\n' >&2; return 2; } + test ! -e "$icu_metadata" && test ! -L "$icu_metadata" || { printf 'reused ICU metadata path\n' >&2; return 2; } + + # Validate all interpolated atoms before assembling a shell command. The + # response URL is the only untrusted interpolation and cannot contain '. + command_text="set -eu + umask 077 + test -d /data/local/tmp/ib-icu && test ! -L /data/local/tmp/ib-icu || exit 78 + test \"\$(sha256sum '$icu_dex' | cut -d ' ' -f 1)\" = '$icu_dex_sha' || exit 78 + test \"\$(sha256sum '$icu_so' | cut -d ' ' -f 1)\" = '$icu_so_sha' || exit 78 + icu_work=\$(mktemp -d /data/local/tmp/ib-icu/capture.XXXXXXXX) || exit 79 + trap 'rm -rf \"\$icu_work\"' EXIT HUP INT TERM + export ICU_CAPTURE_BODY_PATH=\"\$icu_work/body\" + export ICU_CAPTURE_METADATA_PATH=\"\$icu_work/metadata\" + export CLASSPATH='$icu_dex' + app_process -Dicu.library='$icu_so' /system/bin Idric.Generated get '$icu_url' >/dev/null + test -f \"\$ICU_CAPTURE_BODY_PATH\" && test ! -L \"\$ICU_CAPTURE_BODY_PATH\" || exit 80 + test -f \"\$ICU_CAPTURE_METADATA_PATH\" && test ! -L \"\$ICU_CAPTURE_METADATA_PATH\" || exit 80 + metadata_bytes=\$(wc -c < \"\$ICU_CAPTURE_METADATA_PATH\" | tr -d '[:space:]') + body_bytes=\$(wc -c < \"\$ICU_CAPTURE_BODY_PATH\" | tr -d '[:space:]') + test \"\$metadata_bytes\" -gt 0 && test \"\$metadata_bytes\" -le 8192 || exit 81 + test \"\$body_bytes\" -gt 0 && test \"\$body_bytes\" -le 4194304 || exit 81 + printf 'IBICU01 %08d %08d\n' \"\$metadata_bytes\" \"\$body_bytes\" + cat \"\$ICU_CAPTURE_METADATA_PATH\" + cat \"\$ICU_CAPTURE_BODY_PATH\"" + + # Staged packet never enters Arena. App stdout cannot be used as metadata; + # it is discarded by the remote command. A failed rish exits before publish. + local_packet_directory=$(mktemp -d "$icu_output.packet.XXXXXXXX") || return 1 + local_packet="$local_packet_directory/packet" + if ! (umask 077; ulimit -f 8192; ib_shizuku_run_bounded 30 "$command_text" > "$local_packet"); then + rm -rf "$local_packet_directory" + return 1 + fi + packet_header=$(head -c 26 "$local_packet") || { + rm -rf "$local_packet_directory" + return 1 + } + case "$packet_header" in + 'IBICU01 '????????' '????????) ;; + *) rm -rf "$local_packet_directory"; return 1 ;; + esac + packet_sizes=${packet_header#IBICU01 } + metadata_padded=${packet_sizes%% *} + body_padded=${packet_sizes#* } + case "$metadata_padded$body_padded" in *[!0123456789]*) rm -rf "$local_packet_directory"; return 1 ;; esac + metadata_bytes=$(printf '%s' "$metadata_padded" | sed 's/^0*//') + body_bytes=$(printf '%s' "$body_padded" | sed 's/^0*//') + test -n "$metadata_bytes" || metadata_bytes=0 + test -n "$body_bytes" || body_bytes=0 + if test "$metadata_bytes" -le 0 || test "$metadata_bytes" -gt 8192 || + test "$body_bytes" -le 0 || test "$body_bytes" -gt 4194304 || + test "$(wc -c < "$local_packet" | tr -d '[:space:]')" -ne "$((26 + metadata_bytes + body_bytes))"; then + rm -rf "$local_packet_directory" + return 1 + fi + + # Use byte offsets, not line splitting: ICU's body can contain arbitrary + # binary data, including NUL, CRLF and non-UTF-8 bytes. + tail -c +27 "$local_packet" | head -c "$metadata_bytes" > "$local_packet_directory/metadata" || { + rm -rf "$local_packet_directory"; return 1; + } + tail -c +"$((27 + metadata_bytes))" "$local_packet" > "$local_packet_directory/body" || { + rm -rf "$local_packet_directory"; return 1; + } + if test "$(wc -c < "$local_packet_directory/metadata" | tr -d '[:space:]')" -ne "$metadata_bytes" || + test "$(wc -c < "$local_packet_directory/body" | tr -d '[:space:]')" -ne "$body_bytes"; then + rm -rf "$local_packet_directory" + return 1 + fi + mv "$local_packet_directory/metadata" "$icu_metadata" || { + rm -rf "$local_packet_directory"; return 1; + } + mv "$local_packet_directory/body" "$icu_output" || { + rm -f "$icu_metadata" + rm -rf "$local_packet_directory" + return 1 + } + rm -rf "$local_packet_directory" +} diff --git a/native/android/ib_arena_slice.c b/native/android/ib_arena_slice.c new file mode 100644 index 00000000..7341504f --- /dev/null +++ b/native/android/ib_arena_slice.c @@ -0,0 +1,148 @@ +#define _POSIX_C_SOURCE 200809L +#define _FILE_OFFSET_BITS 64 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef O_NOFOLLOW +#error "This reader requires O_NOFOLLOW" +#endif +#ifndef O_DIRECTORY +#error "This reader requires O_DIRECTORY" +#endif + +#define MAX_SOURCE_BYTES (4U * 1024U * 1024U) +#define BUFFER_BYTES 65536U + +static int error(const char *what) { + fprintf(stderr, "ib-arena-slice: %s: %s\n", what, strerror(errno)); + return 1; +} + +static int number(const char *text, uint64_t *out) { + char *end; + uintmax_t parsed; + if (*text == '\0' || *text == '-' || *text == '+') return -1; + errno = 0; + parsed = strtoumax(text, &end, 10); + if (errno != 0 || *end != '\0' || parsed > INT64_MAX) return -1; + *out = (uint64_t)parsed; + return 0; +} + +static char *suffix(const char *path, const char *extension) { + size_t a = strlen(path), b = strlen(extension); + if (a > SIZE_MAX - b - 1) return NULL; + char *name = malloc(a + b + 1); + if (!name) return NULL; + memcpy(name, path, a); + memcpy(name + a, extension, b + 1); + return name; +} + +static int read_used(int fd, uint64_t *used) { + char buffer[64]; + ssize_t n; + do { n = read(fd, buffer, sizeof(buffer) - 1); } + while (n < 0 && errno == EINTR); + if (n <= 0 || n >= (ssize_t)sizeof(buffer) - 1) return -1; + buffer[n] = '\0'; + if (buffer[n - 1] != '\n') return -1; + buffer[n - 1] = '\0'; + return number(buffer, used); +} + +static int write_all(const unsigned char *bytes, size_t length) { + size_t offset = 0; + while (offset < length) { + ssize_t n = write(STDOUT_FILENO, bytes + offset, length - offset); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return -1; + offset += (size_t)n; + } + return 0; +} + +static int dump_slice(const char *arena, uint64_t offset, uint64_t length) { + char *lock_path = suffix(arena, ".lock"); + char *used_path = suffix(arena, ".used"); + int lock = -1, descriptor = -1, used_fd = -1, result = 1; + unsigned char buffer[BUFFER_BYTES]; + struct stat info; + uint64_t committed; + if (!lock_path || !used_path) goto end; + lock = open(lock_path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (lock < 0 || flock(lock, LOCK_SH) != 0) goto end; + descriptor = open(arena, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + used_fd = open(used_path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (descriptor < 0 || used_fd < 0) goto end; + if (fstat(descriptor, &info) != 0 || !S_ISREG(info.st_mode) || + read_used(used_fd, &committed) != 0 || + info.st_size < 0 || committed > (uint64_t)info.st_size || + length > MAX_SOURCE_BYTES || offset > committed || length > committed - offset) { + errno = EINVAL; + goto end; + } + while (length) { + size_t take = length < BUFFER_BYTES ? (size_t)length : BUFFER_BYTES; + ssize_t n = pread(descriptor, buffer, take, (off_t)offset); + if (n < 0 && errno == EINTR) continue; + if (n <= 0 || write_all(buffer, (size_t)n) != 0) { + errno = EIO; + goto end; + } + offset += (uint64_t)n; + length -= (uint64_t)n; + } + result = 0; +end: + if (result) error("invalid or unreadable committed arena slice"); + if (used_fd >= 0) close(used_fd); + if (descriptor >= 0) close(descriptor); + if (lock >= 0) close(lock); + free(lock_path); + free(used_path); + return result; +} + +static int sync_path_and_directory(const char *path) { + int fd = -1, directory_fd = -1, result = 1; + char *parent = strdup(path); + if (!parent) return error("could not allocate parent pathname"); + char *slash = strrchr(parent, '/'); + if (slash == parent) parent[1] = '\0'; + else if (slash) *slash = '\0'; + else { free(parent); parent = strdup("."); } + if (!parent) return error("could not allocate current directory"); + fd = open(path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0 || fsync(fd) != 0) goto done; + directory_fd = open(parent, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (directory_fd < 0 || fsync(directory_fd) != 0) goto done; + result = 0; +done: + if (result) error("cannot sync file and its parent directory"); + if (fd >= 0) close(fd); + if (directory_fd >= 0) close(directory_fd); + free(parent); + return result; +} + +int main(int argc, char **argv) { + uint64_t offset, length; + if (argc == 3 && strcmp(argv[1], "sync") == 0) + return sync_path_and_directory(argv[2]); + if (argc == 5 && strcmp(argv[1], "slice") == 0 && + number(argv[3], &offset) == 0 && number(argv[4], &length) == 0) + return dump_slice(argv[2], offset, length); + fprintf(stderr, "usage: ib-arena-slice slice ARENA OFFSET LENGTH | sync PATH\n"); + return 2; +} diff --git a/tests/icu_native_transport_probe.c b/tests/icu_native_transport_probe.c new file mode 100644 index 00000000..a1cabc1d --- /dev/null +++ b/tests/icu_native_transport_probe.c @@ -0,0 +1,46 @@ +/* CI-only narrow native harness for *actual* ICU #35 transport.c. It is not + * an Android app_process/Idriç replacement or a browser URL parser. */ +#include +#include +#include +#include +#include + +int icu_send_http(const char *, int, const char *, const char *, int); +int icu_send_https(const char *, int, const char *, const char *, int); + +int main(int argc, char **argv) { + if (argc != 2) return 2; + const char *url = argv[1]; + int tls = 0; + if (strncmp(url, "http://", 7) == 0) url += 7; + else if (strncmp(url, "https://", 8) == 0) { + url += 8; + tls = 1; + } else return 2; + const char *colon = strchr(url, ':'); + const char *slash = strchr(url, '/'); + if (!colon || !slash || colon >= slash || colon == url) return 2; + if ((size_t)(colon - url) >= 128U) return 2; + char host[128]; + memcpy(host, url, (size_t)(colon - url)); + host[colon - url] = '\0'; + if (strcmp(host, "localhost") != 0 && + strcmp(host, "127.0.0.1") != 0) return 2; + errno = 0; + char *end = NULL; + long port_long = strtol(colon + 1, &end, 10); + if (errno || end != slash || port_long < 1 || port_long > 65535) + return 2; + int port = (int)port_long; + for (const unsigned char *p = (const unsigned char *)slash; *p; ++p) + if (*p <= 32U || *p >= 127U) return 2; + char request[4096]; + int count = snprintf(request, sizeof(request), + "GET %s HTTP/1.1\r\nHost: %s:%d\r\nConnection: close\r\n\r\n", + slash, host, port); + if (count <= 0 || (size_t)count >= sizeof(request)) return 2; + return tls + ? icu_send_https(host, port, request, NULL, 0) + : icu_send_http(host, port, request, NULL, 0); +} diff --git a/tests/test_android_icu_arena.sh b/tests/test_android_icu_arena.sh new file mode 100644 index 00000000..147297cd --- /dev/null +++ b/tests/test_android_icu_arena.sh @@ -0,0 +1,215 @@ +#!/bin/sh +# Host contract test with real compiled canonical appendfat_arena and native +# reader. The Shizuku + ICU #35 producer is deliberately mocked here. +set -eu +root=$(CDPATH='' cd -- "$(dirname "$0")/.." && pwd) +arena_tool=${IB_APPEND_ARENA:-} +reader=${IB_ARENA_SLICE:-} +test -x "$arena_tool" && test -x "$reader" || { + printf 'IB_APPEND_ARENA and IB_ARENA_SLICE executables required\n' >&2 + exit 2 +} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +mkdir -p "$work/lib" "$work/bin" "$work/test-commands" "$work/mock" +cp "$root/lib/android_shizuku_icu_provider.grease" "$root/lib/android_binder_icu.grease" "$work/lib/" +cp "$root/bin/ib_android_icu_arena.grease" "$work/bin/" +cp "$root/bin/ib_android_arena_read.grease" "$work/bin/" + +# Simulate the *framed* output of a single successful PR #35 ICU invocation. +# The expected remote command must contain both paired capture env variables, +# signed artifact checks and the actual Idric.Generated get path. These tests +# say nothing about a deployed ICU binary, TLS or real rish. +cat > "$work/lib/android_shizuku_host.grease" <<'SHIZUKU' +ib_shizuku_authority() { + if test "${IB_TEST_DENIED:-0}" = 1; then printf 'uid:10123\n' + else printf 'shell\n'; fi +} +ib_shizuku_run_bounded() { + test "$1" = 30 || return 99 + printf '%s\n' "$2" >> "$IB_TEST_REMOTE_COMMANDS" + case "$2" in + *ICU_CAPTURE_BODY_PATH*ICU_CAPTURE_METADATA_PATH*Idric.Generated\ get*) ;; + *) return 91 ;; + esac + case "$2" in *sha256sum*'IBICU01 %08d %08d'*) ;; *) return 92 ;; esac + stub_work=$(mktemp -d "$IB_TEST_TMP/stub.XXXXXXXX") || return 90 + stub_body="$stub_work/body" + stub_meta="$stub_work/meta" + mode=${IB_TEST_MODE:-ok} + status=200 + resolved=https://example.com/final + redirect_count=2 + content_type='text/html; charset=utf-8' + content_encoding=identity + utf8=valid + case "$mode" in + ok|wrong_length|wrong_declared|duplicate|bad_status|bad_resolved|missing) printf 'C67 π ☼\n' > "$stub_body" ;; + binary) printf 'bin\000ary\377\n' > "$stub_body"; content_type=application/octet-stream; utf8=invalid ;; + utf8_invalid) printf 'bad\377' > "$stub_body"; utf8=invalid ;; + no_charset) printf '

no charset

' > "$stub_body"; content_type=text/html ;; + gzip) printf '

gzip marker

' > "$stub_body"; content_encoding=gzip ;; + fail) printf 'partial'; rm -rf "$stub_work"; return 29 ;; + bad_frame) printf 'not-an-ICU-envelope\n'; rm -rf "$stub_work"; return 0 ;; + oversized) head -c 4194305 /dev/zero > "$stub_body" ;; + *) rm -rf "$stub_work"; return 98 ;; + esac + case "$mode" in + bad_status) status=404 ;; + bad_resolved) resolved='javascript:alert(1)' ;; + esac + body_bytes=$(wc -c < "$stub_body" | tr -d '[:space:]') + declared=$body_bytes + received=$body_bytes + case "$mode" in + wrong_length) received=$((body_bytes - 1)) ;; + wrong_declared) declared=$((body_bytes - 1)) ;; + esac + { + printf 'http_status\t%s\n' "$status" + printf 'content_type\t%s\n' "$content_type" + printf 'content_encoding\t%s\n' "$content_encoding" + printf 'declared_length\t%s\n' "$declared" + printf 'received_length\t%s\n' "$received" + printf 'utf8\t%s\n' "$utf8" + printf 'resolved_url\t%s\n' "$resolved" + printf 'redirect_count\t%s\n' "$redirect_count" + if test "$mode" = duplicate; then printf 'http_status\t200\n'; fi + } > "$stub_meta" + if test "$mode" = missing; then + sed -i '/^redirect_count/d' "$stub_meta" + fi + meta_bytes=$(wc -c < "$stub_meta" | tr -d '[:space:]') + printf 'IBICU01 %08d %08d\n' "$meta_bytes" "$body_bytes" + cat "$stub_meta" + cat "$stub_body" + rm -rf "$stub_work" +} +SHIZUKU + +export IB_TEST_TMP="$work/mock" +export IB_TEST_REMOTE_COMMANDS="$work/test-commands/commands.txt" +export IB_C67_ICU_DEX=/data/local/tmp/ib-icu/classes.dex +export IB_C67_ICU_SO=/data/local/tmp/ib-icu/libicu_transport.so +export IB_C67_ICU_DEX_SHA256=1111111111111111111111111111111111111111111111111111111111111111 +export IB_C67_ICU_SO_SHA256=2222222222222222222222222222222222222222222222222222222222222222 +export IB_APPEND_ARENA="$arena_tool" +export IB_ARENA_SLICE="$reader" +mkdir -p "$work/private" +chmod 700 "$work/private" +"$arena_tool" create "$work/private/responses.arena" 16777216 >/dev/null +run() { sh "$work/bin/ib_android_icu_arena.grease" "$1" "$2" "$work/private" "${3:-1}"; } +readback() { sh "$work/bin/ib_android_arena_read.grease" "$1" "$work/private"; } +field() { awk -F '\t' -v name="$2" '$1==name {print $2}' "$work/private/receipts/$1.tsv"; } + +run first https://example.com/start 17 > "$work/output1" +test "$(field first schema)" = ib-icu-arena-v2 +test "$(field first http_status)" = 200 +test "$(field first resolved_url)" = https://example.com/final +test "$(field first redirect_count)" = 2 +test "$(field first mime_type)" = text/html +test "$(field first charset)" = utf-8 +test "$(field first source_generation)" = 17 +test "$(field first source_key)" = arena:first +test "$(field first acquisition_complete)" = YES +test "$(field first navigation_admitted)" = YES +first_bytes=$(field first byte_count) +test "$(field first arena_offset)" = 0 +test "$(cat "$work/private/responses.arena.used")" -gt 0 +printf 'C67 π ☼\n' > "$work/expected" +readback first > "$work/first-body" +cmp -s "$work/expected" "$work/first-body" + +export IB_TEST_MODE=binary +run second https://example.com/binary 18 > "$work/output2" +test "$(field second navigation_admitted)" = NO +test "$(field second mime_type)" = application/octet-stream +test "$(field second utf8)" = invalid +printf 'bin\000ary\377\n' > "$work/expected-binary" +readback second > "$work/read2" +cmp -s "$work/expected-binary" "$work/read2" + +# Correct non-HTML, unsupported encoding, absent charset and invalid UTF-8 +# remain readable as bounded bytes but never acquire navigation authority. +for mode in no_charset gzip utf8_invalid; do + export IB_TEST_MODE="$mode" + run "$mode" https://example.com/ > /dev/null + test "$(field "$mode" navigation_admitted)" = NO + readback "$mode" > "$work/$mode.read" + test -s "$work/$mode.read" +done + +# Mutated receipts must never emit any prefix. Also refuse forged admission +# on an otherwise valid non-HTML committed range. +cp "$work/private/receipts/first.tsv" "$work/first.original.tsv" +sed -i 's/^sha256[[:space:]].*/sha256\t0000000000000000000000000000000000000000000000000000000000000000/' \ + "$work/private/receipts/first.tsv" +if readback first > "$work/read-invalid" 2>/dev/null; then exit 1; fi +test ! -s "$work/read-invalid" +cp "$work/first.original.tsv" "$work/private/receipts/first.tsv" +cp "$work/private/receipts/second.tsv" "$work/second.original.tsv" +sed -i 's/^navigation_admitted[[:space:]].*/navigation_admitted\tYES/' \ + "$work/private/receipts/second.tsv" +if readback second > "$work/forged-admission" 2>/dev/null; then exit 1; fi +test ! -s "$work/forged-admission" +cp "$work/second.original.tsv" "$work/private/receipts/second.tsv" + +# v1 can still supply verified bytes but cannot be upgraded to navigation. +{ + printf 'schema\tib-icu-arena-v1\n' + printf 'request_id\tlegacy\n' + printf 'requested_url\t%s\n' "$(field first requested_url)" + printf 'resolved_url\tUNOBSERVED\nhttp_status\tUNOBSERVED\nmime_type\tUNOBSERVED\n' + printf 'transport\ticu-app-process-via-binder-shizuku\narena_name\tresponses.arena\n' + printf 'arena_offset\t%s\n' "$(field first arena_offset)" + printf 'arena_end\t%s\n' "$(field first arena_end)" + printf 'byte_count\t%s\n' "$(field first byte_count)" + printf 'sha256\t%s\n' "$(field first sha256)" + printf 'reopen_check\tMATCH\nnavigation_admitted\tNO\n' +} > "$work/private/receipts/legacy.tsv" +readback legacy > "$work/legacy-body" +cmp -s "$work/expected" "$work/legacy-body" + +# None of these producer failures may advance the commit pointer or publish a +# receipt. Fresh request IDs are used because request locks are single-use. +used=$(cat "$work/private/responses.arena.used") +expect_refusal() { + mode=$1 + export IB_TEST_MODE="$mode" + if run "reject_$mode" https://example.com/ > "$work/refused" 2>/dev/null; then exit 1; fi + test ! -e "$work/private/receipts/reject_$mode.tsv" + test "$(cat "$work/private/responses.arena.used")" = "$used" +} +for mode in fail bad_frame oversized wrong_length wrong_declared duplicate bad_status bad_resolved missing; do + expect_refusal "$mode" +done +export IB_TEST_MODE=ok +if run second https://example.com/ > /dev/null 2>&1; then exit 1; fi +if run zero https://example.com/ 0 > /dev/null 2>&1; then exit 1; fi +if run leading https://example.com/ 017 > /dev/null 2>&1; then exit 1; fi + +# Denied Shizuku or unsafe URLs must not run a remote command. +export IB_TEST_DENIED=1 +before=$(wc -l < "$IB_TEST_REMOTE_COMMANDS") +if run denied https://example.com/ > /dev/null 2>&1; then exit 1; fi +test "$(wc -l < "$IB_TEST_REMOTE_COMMANDS")" = "$before" +unset IB_TEST_DENIED +if run injected "https://example.com/a'\$(touch '$work/INJECTED')" > /dev/null 2>&1; then exit 1; fi +test ! -e "$work/INJECTED" +test "$(wc -l < "$IB_TEST_REMOTE_COMMANDS")" = "$before" +if run unsupported file:///etc/passwd > /dev/null 2>&1; then exit 1; fi +test "$(cat "$work/private/responses.arena.used")" = "$used" + +# Independent native reader guards out-of-range and malformed used pointers. +if "$reader" slice "$work/private/responses.arena" "$used" 1 >/dev/null 2>&1; then exit 1; fi +if "$reader" slice "$work/private/responses.arena" 0 4194305 >/dev/null 2>&1; then exit 1; fi +cp "$work/private/responses.arena.used" "$work/used.bak" +printf '99999999999999999\n' > "$work/private/responses.arena.used" +if "$reader" slice "$work/private/responses.arena" 0 1 >/dev/null 2>&1; then exit 1; fi +cp "$work/used.bak" "$work/private/responses.arena.used" +"$reader" slice "$work/private/responses.arena" 0 "$first_bytes" > "$work/restored" +cmp -s "$work/expected" "$work/restored" + +printf '%s\n' \ + 'PASS: framed ICU #35 metadata/body contract, observed redirect, canonical range/digest, v1 refusal, conditional navigation, binary fidelity and negative admission checks' \ + 'NOT_RUN: real ICU HTTPS, Android DEX/ARM32, physical Shizuku, C67 UI, private PFD or durable navigation' diff --git a/tests/test_android_icu_native_bridge.sh b/tests/test_android_icu_native_bridge.sh new file mode 100644 index 00000000..fd581a71 --- /dev/null +++ b/tests/test_android_icu_native_bridge.sh @@ -0,0 +1,232 @@ +#!/bin/sh +# Host integration of actual ICU #35 native transport through IB's unchanged +# Shizuku framing + canonical Arena producer and verified reader. +# The app_process wrapper, Shizuku authority and HTTP server are TEST ONLY. +# This does not qualify Idriç DEX, real rish, Android or physical C67. +set -eu +export LC_ALL=C +root=$(CDPATH='' cd -- "$(dirname "$0")/.." && pwd) +arena_tool=${IB_APPEND_ARENA:-} +slice_tool=${IB_ARENA_SLICE:-} +probe=${IB_ICU_NATIVE_PROBE:-} +native_so=${IB_ICU_NATIVE_SO:-} +for executable in "$arena_tool" "$slice_tool" "$probe"; do + test -n "$executable" && test -x "$executable" || { + echo 'compiled native test prerequisites absent' >&2 + exit 2 + } +done +test -n "$native_so" && test -f "$native_so" || exit 2 +work=$(mktemp -d) +server_pid= +cleanup() { + if test -n "$server_pid"; then kill "$server_pid" 2>/dev/null || :; fi + rm -rf "$work" +} +trap cleanup EXIT HUP INT TERM +mkdir -p "$work/lib" "$work/bin" "$work/fake-bin" "$work/private-shell" "$work/remote-commands" +cp "$root/lib/android_shizuku_icu_provider.grease" "$root/lib/android_binder_icu.grease" "$work/lib/" +cp "$root/bin/ib_android_icu_arena.grease" "$root/bin/ib_android_arena_read.grease" "$work/bin/" +cp "$native_so" "$work/private-shell/libicu_transport.so" +printf '%s\n' 'fixture-not-real-android-dex' > "$work/private-shell/classes.dex" +chmod 700 "$work/private-shell" +export IB_ICU_TEST_SHELL_ROOT="$work/private-shell" +export IB_TEST_REMOTE_COMMANDS="$work/remote-commands/commands.txt" +export IB_ICU_NATIVE_PROBE="$probe" +export IB_C67_ICU_DEX=/data/local/tmp/ib-icu/classes.dex +export IB_C67_ICU_SO=/data/local/tmp/ib-icu/libicu_transport.so +export IB_C67_ICU_DEX_SHA256 +IB_C67_ICU_DEX_SHA256=$(sha256sum "$work/private-shell/classes.dex" | cut -d ' ' -f 1) +export IB_C67_ICU_SO_SHA256 +IB_C67_ICU_SO_SHA256=$(sha256sum "$work/private-shell/libicu_transport.so" | cut -d ' ' -f 1) + +cat > "$work/lib/android_shizuku_host.grease" <<'SHIZUKU' +ib_shizuku_authority() { printf 'shell\n'; } +ib_shizuku_run_bounded() { + test "$1" = 30 || return 90 + case "$2" in + *ICU_CAPTURE_BODY_PATH*ICU_CAPTURE_METADATA_PATH*Idric.Generated\ get*) ;; + *) return 91 ;; + esac + printf '%s\n' "$2" >> "$IB_TEST_REMOTE_COMMANDS" + # Host-only translation of the Android shell-owned private directory. + # The production provider still requires the pinned Android path atoms. + translated=$(printf '%s\n' "$2" | + sed "s|/data/local/tmp/ib-icu|$IB_ICU_TEST_SHELL_ROOT|g") + sh -c "$translated" +} +SHIZUKU +cat > "$work/fake-bin/app_process" <<'WRAPPER' +#!/bin/sh +# This wrapper invokes ICU #35's *actual compiled native transport*, but is +# deliberately not a claim of running a real Android Idriç app_process DEX. +set -eu +test "$#" = 5 +case "$1" in -Dicu.library=*) ;; *) exit 80 ;; esac +test "$2" = /system/bin +test "$3" = Idric.Generated +test "$4" = get +exec "$IB_ICU_NATIVE_PROBE" "$5" +WRAPPER +chmod 700 "$work/fake-bin/app_process" +PATH="$work/fake-bin:$PATH" +export PATH + +http_port=18932 +tls_port=18933 +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$work/server.key" -out "$work/server.crt" -days 1 \ + -subj '/CN=localhost' -addext 'subjectAltName=DNS:localhost' \ + > /dev/null 2>&1 +export SSL_CERT_FILE="$work/server.crt" +python3 - "$http_port" "$tls_port" "$work/server.crt" "$work/server.key" \ + > "$work/server.log" 2>&1 <<'PY' & +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import ssl +import sys +import threading + +plain_port, tls_port = map(int, sys.argv[1:3]) +cert_file, key_file = sys.argv[3:5] + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.0" + + def log_message(self, *args): + pass + + def do_GET(self): + if self.path == "/start": + self.send_response(302) + self.send_header("Location", "/final") + self.end_headers() + return + if self.path == "/not-found": + self.send_response(404) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", "0") + self.end_headers() + return + if self.path == "/truncated": + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", "50") + self.end_headers() + self.wfile.write(b"short") + return + if self.path == "/binary": + body = bytes([0, 255, 10]) + mime = "application/octet-stream" + elif self.path == "/final": + body = b"

ICU 35 host

\n" + mime = "text/html; charset=utf-8" + else: + self.send_response(400) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", mime) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + +plain = ThreadingHTTPServer(("127.0.0.1", plain_port), Handler) +secure = ThreadingHTTPServer(("127.0.0.1", tls_port), Handler) +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain(cert_file, key_file) +secure.socket = context.wrap_socket(secure.socket, server_side=True) +threading.Thread(target=plain.serve_forever, daemon=True).start() +threading.Thread(target=secure.serve_forever, daemon=True).start() +threading.Event().wait() +PY +server_pid=$! +ready=NO +for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do + if curl --max-time 1 -fsS "http://127.0.0.1:$http_port/final" >/dev/null 2>&1; then + ready=YES + break + fi + sleep 1 +done +test "$ready" = YES || { cat "$work/server.log" >&2; exit 1; } + +export IB_APPEND_ARENA="$arena_tool" IB_ARENA_SLICE="$slice_tool" +mkdir -m 700 "$work/private" +"$arena_tool" create "$work/private/responses.arena" 16777216 >/dev/null +run() { sh "$work/bin/ib_android_icu_arena.grease" "$1" "$2" "$work/private" "$3"; } +readback() { sh "$work/bin/ib_android_arena_read.grease" "$1" "$work/private"; } +field() { awk -F '\t' -v name="$2" '$1==name {print $2}' "$work/private/receipts/$1.tsv"; } + +# Both HTTPS and HTTP use the actual compiled ICU socket/TLS response parser, +# through IB's remote length-framed capture and verified canonical append. +run secure "https://localhost:$tls_port/start" 17 > "$work/out-secure" +test "$(field secure schema)" = ib-icu-arena-v2 +test "$(field secure resolved_url)" = "https://localhost:$tls_port/final" +test "$(field secure redirect_count)" = 1 +test "$(field secure http_status)" = 200 +test "$(field secure mime_type)" = text/html +test "$(field secure charset)" = utf-8 +test "$(field secure navigation_admitted)" = YES +printf '

ICU 35 host

\n' > "$work/expected.html" +readback secure > "$work/secure.body" +cmp -s "$work/expected.html" "$work/secure.body" +# A separate AI-CI program validates the *actual* persisted ICU response, +# with its own byte-offset read rather than IB's native slice/checking logic. +aici_verify=${IB_AICI_SOURCE_VERIFY:-} +test -n "$aici_verify" && test -f "$aici_verify" || { + printf 'independent AI-CI source verifier must be pinned\n' >&2 + exit 1 +} +sh "$aici_verify" "$work/private" secure 17 "https://localhost:$tls_port/start" \ + > "$work/aici-secure.tsv" +grep -F "$(printf 'PASS\tib-source-v2-admitted\tsecure\t17\t')" "$work/aici-secure.tsv" + +run plain "http://127.0.0.1:$http_port/start" 18 > "$work/out-plain" +test "$(field plain resolved_url)" = "http://127.0.0.1:$http_port/final" +test "$(field plain navigation_admitted)" = YES +readback plain > "$work/plain.body" +cmp -s "$work/expected.html" "$work/plain.body" +sh "$aici_verify" "$work/private" plain 18 "http://127.0.0.1:$http_port/start" \ + > "$work/aici-plain.tsv" +grep -F "$(printf 'PASS\tib-source-v2-admitted\tplain\t18\t')" "$work/aici-plain.tsv" + +run binary "http://127.0.0.1:$http_port/binary" 19 > "$work/out-binary" +test "$(field binary utf8)" = invalid +test "$(field binary navigation_admitted)" = NO +printf '\000\377\n' > "$work/expected-binary" +readback binary > "$work/binary.body" +cmp -s "$work/expected-binary" "$work/binary.body" +if sh "$aici_verify" "$work/private" binary 19 \ + "http://127.0.0.1:$http_port/binary" > "$work/aici-binary.tsv" 2>/dev/null; then + echo 'independent oracle wrongly admitted binary resource' >&2 + exit 1 +fi +test ! -s "$work/aici-binary.tsv" + +# ICU refuses incomplete and 404 acquisitions before any Arena publication. +used=$(cat "$work/private/responses.arena.used") +for mode in not-found truncated; do + if run "reject_$mode" "https://localhost:$tls_port/$mode" 20 \ + > "$work/reject.stdout" 2> "$work/reject.stderr"; then + echo "ICU native $mode unexpectedly succeeded" >&2 + exit 1 + fi + test ! -e "$work/private/receipts/reject_$mode.tsv" + test "$(cat "$work/private/responses.arena.used")" = "$used" +done +good_digest=$IB_C67_ICU_SO_SHA256 +IB_C67_ICU_SO_SHA256=0000000000000000000000000000000000000000000000000000000000000000 +export IB_C67_ICU_SO_SHA256 +if run wronghash "http://127.0.0.1:$http_port/final" 21 >/dev/null 2>&1; then + echo 'failed ICU binary pin was accepted' >&2 + exit 1 +fi +test ! -e "$work/private/receipts/wronghash.tsv" +test "$(cat "$work/private/responses.arena.used")" = "$used" +IB_C67_ICU_SO_SHA256=$good_digest +export IB_C67_ICU_SO_SHA256 +printf '%s\n' \ + 'PASS: real compiled pinned ICU #35 native HTTP and HTTPS+redirect captured by IB Shizuku frame into canonical appendfat and verified v2 reader' \ + 'PASS: independent AI-CI verified two actual persisted HTML sources and rejected binary resource' \ + 'PASS: binary fidelity, HTTP 404, truncated length, native hash mismatch, denied publication' \ + 'NOT_RUN: actual Android app_process Idric DEX, physical Shizuku/rish, Binder/PFD and C67 UI'