[DRAFT] Convert admitted recovered HTML into typed Prepaint Version 1

.github/workflows/ib-html-prepaint-static.yml

@@ -0,0 +1,31 @@
+name: IB independent Prepaint Version 1 oracle
+
+on:
+  pull_request:
+    paths:
+      - 'src/IB/HtmlPrepaint.idric'
+      - 'src/IB/HtmlLinks.idric'
+      - 'src/IB/Information.idric'
+      - 'src/IB/RecoveredInformation.idric'
+      - 'tests/prepaint_v1_oracle.c'
+      - 'tests/test_prepaint_oracle_static.sh'
+      - 'tests/fixtures/ib-html/**'
+      - '.github/workflows/ib-html-prepaint-static.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  independent-wire:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+      - name: Independently parse frozen Version 1 witnesses and reject mutants
+        run: |
+          test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}"
+          sh -n tests/test_prepaint_oracle_static.sh
+          sh tests/test_prepaint_oracle_static.sh

.github/workflows/idric-core.yml

@@ -101,6 +101,115 @@ jobs:
           grep -Fx 'nonincreasing-revision-refused=True' /tmp/ib-canonical-information.txt
           grep -Fx 'version1=serialized' /tmp/ib-canonical-information.txt
 
+      - name: Compile and exercise admitted HTML to Prepaint Version 1
+        working-directory: src
+        run: |
+          /tmp/idric/bin/idris2 HtmlPrepaintSmoke.idric -o ib-html-prepaint-smoke
+          test -x build/exec/ib-html-prepaint-smoke
+          build/exec/ib-html-prepaint-smoke | tee /tmp/html-prepaint-smoke.txt
+          grep -Fx 'version1=True' /tmp/html-prepaint-smoke.txt
+          grep -Fx 'heading-six=True' /tmp/html-prepaint-smoke.txt
+          grep -Fx 'digest-mismatch-refused=True' /tmp/html-prepaint-smoke.txt
+          /tmp/idric/bin/idris2 HtmlPrepaintCommand.idric -o ib-html-prepaint
+          test -x build/exec/ib-html-prepaint
+
+      - name: Check out independently accepted AI-CI Prepaint consumer
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          repository: isomorphisms/ai-ci
+          ref: 0fb20839a2da68be0aafc10511e13c634bfa02b9
+          path: .tools/ai-ci-prepaint
+          persist-credentials: false
+
+      - name: Independently verify actual Idriç output and source-bound mutations
+        env:
+          IB_AICI_PREPAINT_VERIFIER: ${{ github.workspace }}/.tools/ai-ci-prepaint/ib-prepaint/verify.sh
+          IB_HTML_PREPAINT_RECEIPT_ROOT: ${{ runner.temp }}/ib-html-prepaint-receipts
+        run: |
+          test "$(git -C .tools/ai-ci-prepaint rev-parse HEAD)" = 0fb20839a2da68be0aafc10511e13c634bfa02b9
+          sh -n bin/ib_emit_html_prepaint.grease
+          sh -n bin/ib_arena_to_html_prepaint.grease
+          sh -n tests/test_arena_html_refusals.sh
+          sh tests/test_arena_html_refusals.sh
+          sh -n tests/test_html_prepaint.sh
+          sh .tools/ai-ci-prepaint/ib-prepaint/self-test.sh
+          sh tests/test_html_prepaint.sh
+          {
+            printf 'repository\tisomorphisms/ib\n'
+            printf 'source_commit\t%s\n' "$(git rev-parse HEAD)"
+            printf 'idric_commit\t%s\n' "$IDRIC_REF"
+            printf 'independent_ai_ci_commit\t0fb20839a2da68be0aafc10511e13c634bfa02b9\n'
+            printf 'compiler_version\t%s\n' "$(/tmp/idric/bin/idris2 --version)"
+            printf 'platform\t%s\n' "$(uname -sm)"
+          } > "$IB_HTML_PREPAINT_RECEIPT_ROOT/inputs.tsv"
+          sha256sum src/build/exec/ib-html-prepaint src/build/exec/ib-html-prepaint-smoke \
+            tests/prepaint_v1_oracle.c src/IB/HtmlPrepaint.idric src/IB/HtmlLinks.idric \
+            > "$IB_HTML_PREPAINT_RECEIPT_ROOT/artifact-inputs.sha256"
+          (cd "$IB_HTML_PREPAINT_RECEIPT_ROOT" && sha256sum *.prepaint > wire.sha256)
+
+      - name: Retain independently checked fixture receipts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: ib-html-prepaint-idric-and-aici
+          path: ${{ runner.temp }}/ib-html-prepaint-receipts/
+          if-no-files-found: error
+
+      - name: Check out verified Arena v2 acquisition and reader implementation
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          repository: isomorphisms/ib
+          ref: e2a2a17a4378b7c3845ec43f42d1d3360b12f051
+          path: .deps/ib-arena
+          persist-credentials: false
+
+      - name: Check out canonical append-fat userspace Arena
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          repository: isomorphisms/sd-card-append-fat
+          ref: 05a0cb7246c5a9f46abfbe92925812e7b607f454
+          path: .deps/appendfat
+          persist-credentials: false
+
+      - name: Resolve qualified host ICK compiler
+        uses: isomorphisms/ai-ci/ick-host@015cc7901ae0b3ad262b476f24e129b53c56db95
+        id: ick
+
+      - name: Reopen real canonical Arena bytes and compile with actual Idriç converter
+        env:
+          ICK: ${{ steps.ick.outputs.compiler }}
+          IB_AICI_PREPAINT_VERIFIER: ${{ github.workspace }}/.tools/ai-ci-prepaint/ib-prepaint/verify.sh
+          IB_HTML_PREPAINT_RECEIPT_ROOT: ${{ runner.temp }}/ib-html-prepaint-receipts
+        run: |
+          test "$(git -C .deps/ib-arena rev-parse HEAD)" = e2a2a17a4378b7c3845ec43f42d1d3360b12f051
+          test "$(git -C .deps/appendfat rev-parse HEAD)" = 05a0cb7246c5a9f46abfbe92925812e7b607f454
+          mkdir -p build/ib-html-arena
+          "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+            .deps/appendfat/lib/appendfat_arena.c \
+            .deps/appendfat/tools/appendfat_arena.c \
+            -o build/ib-html-arena/appendfat_arena
+          cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+            .deps/ib-arena/native/android/ib_arena_slice.c \
+            -o build/ib-html-arena/ib-arena-slice
+          cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \
+            tests/prepaint_v1_oracle.c -o build/ib-html-arena/prepaint-oracle
+          export IB_APPEND_ARENA="$GITHUB_WORKSPACE/build/ib-html-arena/appendfat_arena"
+          export IB_ARENA_SLICE="$GITHUB_WORKSPACE/build/ib-html-arena/ib-arena-slice"
+          export IB_HTML_PREPAINT_EXEC="$GITHUB_WORKSPACE/src/build/exec/ib-html-prepaint"
+          export IB_PREPAINT_ORACLE="$GITHUB_WORKSPACE/build/ib-html-arena/prepaint-oracle"
+          sh -n tests/test_arena_v2_to_html_prepaint.sh
+          sh tests/test_arena_v2_to_html_prepaint.sh
+          sha256sum build/ib-html-arena/appendfat_arena build/ib-html-arena/ib-arena-slice \
+            build/ib-html-arena/prepaint-oracle \
+            > "$IB_HTML_PREPAINT_RECEIPT_ROOT/arena-host-executables.sha256"
+          (cd "$IB_HTML_PREPAINT_RECEIPT_ROOT" && sha256sum *.prepaint > wire.sha256)
+
+      - name: Retain qualified Arena-v2 mocked-origin integration evidence
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: ib-html-prepaint-arena-v2-mock
+          path: ${{ runner.temp }}/ib-html-prepaint-receipts/
+          if-no-files-found: error
+
       - name: Exercise browser-owned Idriç core
         run: sh bin/ci_browser_foundation.grease exercise-core
 

README.md

@@ -19,6 +19,7 @@ heap, form, and reconstruction outcomes separately.  See
 ## Design notes
 
 - `docs/architecture.md` — ownership and replaceable-service boundaries
+- `docs/source-byte-decoding.md` — source-bound HTML decoding without changing admission
 - `docs/personal-workbench.md` — personal scope, task frontend, user stories, and latency targets
 - `docs/prefetch-and-reading.md` — durable investigation frontiers, disposable fetches, and `~/reading`
 - `docs/tab-categorization.md` — overlapping personal categories and adaptive refinement

bin/ib_arena_to_html_prepaint.grease

@@ -0,0 +1,81 @@
+#!/bin/sh
+# Adapter ONLY for independently reopened committed PR #152 Arena ranges.
+# It requires future admitted ICU metadata; legacy UNOBSERVED v1 refuses.
+# The transport + store remain owned by #152; this imports no replacement.
+set -eu
+umask 077
+fail() { printf 'ib Arena → HTML prepaint: %s\n' "$1" >&2; exit 1; }
+test "$#" -eq 4 || fail 'usage: ib_arena_to_html_prepaint.grease REQUEST_ID PRIVATE_ARENA_ROOT EXPECTED_GENERATION OUTPUT'
+request=$1
+root=$2
+generation=$3
+output=$4
+case "$request" in
+  ''|.|..|*[!abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-]*) fail 'bad request identity' ;;
+esac
+case "$root" in /*) ;; *) fail 'Arena root must be absolute' ;; esac
+test -d "$root" && test ! -L "$root" || fail 'Arena root unavailable or linked'
+root=$(CDPATH='' cd -P -- "$root" && pwd -P)
+manifest="$root/receipts/$request.tsv"
+test -f "$manifest" && test ! -L "$manifest" || fail 'no committed receipt'
+test "$(wc -c < "$manifest" | tr -d '[:space:]')" -le 8192 || fail 'oversized producer receipt'
+awk -F '\t' '
+  NF != 2 || $1 !~ /^[a-z_][a-z_0-9]*$/ || $2 == "" || /\r/ { bad=1 }
+  { if (seen[$1]++) bad=1 }
+  END { if (bad || NR < 14) exit 1 }
+' "$manifest" || fail 'invalid or duplicate producer receipt keys'
+field() { awk -F '\t' -v key="$1" '$1 == key { print $2 }' "$manifest"; }
+test "$(field request_id)" = "$request" || fail 'request mismatch'
+test "$(field reopen_check)" = MATCH || fail 'source not independently reopened'
+test "$(field navigation_admitted)" = YES || fail 'navigation not admitted'
+test "$(field source_generation)" = "$generation" || fail 'out-of-order or missing generation'
+test "$(field acquisition_complete)" = YES || fail 'incomplete ICU acquisition'
+test "$(field mime_type)" = text/html || fail 'unadmitted content type'
+case "$(field charset)" in utf-8|UTF-8) ;; *) fail 'unobserved/unsupported charset' ;; esac
+test "$(field http_status)" = 200 || fail 'unadmitted status'
+requested=$(field requested_url)
+resolved=$(field resolved_url)
+test -n "$requested" && test -n "$resolved" && test "$resolved" != UNOBSERVED ||
+  fail 'actual ICU final URL missing'
+digest=$(field sha256)
+length=$(field byte_count)
+test -n "$digest" && test -n "$length" || fail 'source digest/length missing'
+case "$digest" in ''|*[!0123456789abcdef]*) fail 'source digest invalid' ;; esac
+test "$(printf %s "$digest" | wc -c | tr -d '[:space:]')" -eq 64 || fail 'source digest length invalid'
+case "$length" in ''|*[!0123456789]*) fail 'source length invalid' ;; esac
+test "$length" -gt 0 && test "$length" -le 4194304 || fail 'source length outside bound'
+expected_key="arena:$request"
+test "$(field source_key)" = "$expected_key" || fail 'untrusted source key'
+reader=$(printenv IB_ARENA_READ_EXEC 2>/dev/null || :)
+test -n "$reader" && test -x "$reader" || fail 'PR #152 verified Arena reader executable required'
+parent=$(dirname -- "$output")
+test -d "$parent" && test ! -L "$parent" || fail 'disposable output directory unavailable'
+work=$(mktemp -d "$parent/.ib-html-arena.XXXXXXXX") || fail 'cannot stage retained source'
+cleanup() { rm -rf "$work"; }
+trap cleanup EXIT HUP INT TERM
+# This MUST be PR #152's verified range-reading implementation. It performs
+# the canonical offset, used pointer, length and SHA check before any emission.
+"$reader" "$request" "$root" > "$work/body" ||
+  fail 'canonical committed readback refused'
+test "$(wc -c < "$work/body" | tr -d '[:space:]')" = "$length" ||
+  fail 'reopened committed source length changed'
+test "$(sha256sum "$work/body" | cut -d ' ' -f 1)" = "$digest" ||
+  fail 'reopened committed source digest changed'
+{
+  printf 'schema\tib-html-admission-v1\n'
+  printf 'source_kind\tarena\n'
+  printf 'source_key\t%s\n' "$expected_key"
+  printf 'source_generation\t%s\n' "$generation"
+  printf 'requested_url\t%s\n' "$requested"
+  printf 'resolved_url\t%s\n' "$resolved"
+  printf 'http_status\t200\n'
+  printf 'mime_type\ttext/html\n'
+  printf 'charset\t%s\n' "$(field charset)"
+  printf 'sha256\t%s\n' "$digest"
+  printf 'byte_count\t%s\n' "$length"
+  printf 'acquisition_complete\tYES\n'
+  printf 'arena_reopen\tMATCH\n'
+} > "$work/admitted.tsv"
+IB_EXPECTED_SOURCE_GENERATION="$generation" \
+  sh "$(dirname -- "$0")/ib_emit_html_prepaint.grease" \
+  "$work/admitted.tsv" "$work/body" "$expected_key" "$output"

bin/ib_decode_html_bytes.grease

@@ -0,0 +1,129 @@
+#!/usr/bin/env grease
+# Source-bound diagnostic decoding, not a canonical or reader-admission path.
+umask 077
+var owned_temporaries = []
+trap --add EXIT {
+  for temporary in (owned_temporaries) { rm -rf -- $temporary }
+}
+
+proc fail(message) {
+  write -- "ib source decoding: $message" >&2
+  exit 2
+}
+
+proc checked_path(path) {
+  if (not path.startsWith('/')) { fail 'path must be absolute' }
+  var current = path
+  while (current !== '/') {
+    if test -L $current { fail 'path crosses a symlink' }
+    setvar current = $(dirname -- $current)
+  }
+}
+
+proc regular_file(path, limit) {
+  checked_path $path
+  if test ! -f $path { fail 'regular input file unavailable' }
+  var count = $(wc -c < $path)
+  if (int(count) > limit) { fail 'file exceeds decoding bound' }
+}
+
+proc digest(path) {
+  sha256sum -- $path | cut -d ' ' -f 1
+}
+
+proc field(path, key) {
+  awk -F b'\t' -v key=$key '$1 == key { print $2 }' $path
+}
+
+if (len(ARGV) !== 4) {
+  fail 'usage: ib_decode_html_bytes.grease IDRIC_DECODER SOURCE.tsv RAW_BYTES NEW_DERIVATIVE_DIRECTORY'
+}
+var decoder = ARGV[0]
+var supplied_metadata = ARGV[1]
+var supplied_raw = ARGV[2]
+var destination = ARGV[3]
+checked_path $decoder
+if test ! -x $decoder { fail 'compiled Idriç decoder required' }
+regular_file $supplied_metadata 8192
+regular_file $supplied_raw 4194304
+checked_path $destination
+if test -e $destination { fail 'derivative destination already exists' }
+var parent = $(dirname -- $destination)
+if test ! -d $parent { fail 'derivative parent unavailable' }
+var stage = $(mktemp -d "$parent/.ib-decode.XXXXXX")
+call owned_temporaries->append(stage)
+
+# Freeze bounded inputs before interpreting claims or invoking the decoder.
+# The output retains this descriptor exactly; extra source fields are opaque
+# provenance, not an extension to this command's admission authority.
+head -c 8193 -- $supplied_metadata > "$stage/source.tsv"
+head -c 4194305 -- $supplied_raw > "$stage/input.raw"
+regular_file "$stage/source.tsv" 8192
+regular_file "$stage/input.raw" 4194304
+if ! awk -F b'\t' '
+  BEGIN {
+    split("source_key source_generation sha256 byte_count charset navigation_admitted", keys, " ")
+    for (i in keys) required[keys[i]]=1
+  }
+  NF!=2 || $1 !~ /^[a-z_][a-z_0-9]*$/ || $2=="" || /\r/ || seen[$1]++ {bad=1}
+  END {for (key in required) if (!seen[key]) bad=1; exit bad}
+' "$stage/source.tsv" { fail 'malformed or incomplete source descriptor' }
+var key = $(field "$stage/source.tsv" source_key)
+var generation = $(field "$stage/source.tsv" source_generation)
+if (len(key) > 256 or len(generation) > 256) { fail 'source identity exceeds bound' }
+var expected_hash = $(field "$stage/source.tsv" sha256)
+var expected_count = $(field "$stage/source.tsv" byte_count)
+try { printf '%s\t%s\n' $expected_hash $expected_count | awk -F b'\t' '
+  $1 !~ /^[0-9a-f]+$/ || length($1)!=64 ||
+  $2 !~ /^(0|[1-9][0-9]*)$/ || length($2)>7 || $2>4194304 {exit 1}
+' }
+if failed { fail 'invalid source digest or byte count' }
+var navigation = $(field "$stage/source.tsv" navigation_admitted)
+if (navigation !== 'YES' and navigation !== 'NO') { fail 'invalid original navigation observation' }
+var actual_count = $(wc -c < "$stage/input.raw" | tr -d '[:space:]')
+var actual_hash = $(digest "$stage/input.raw")
+if (actual_count !== expected_count) { fail 'source byte count mismatch' }
+if (actual_hash !== expected_hash) { fail 'source digest mismatch' }
+var charset = $(field "$stage/source.tsv" charset)
+var source_metadata_hash = $(digest "$stage/source.tsv")
+
+try { $decoder $charset "$stage/input.raw" "$stage/decoded.utf8" > "$stage/producer.tsv" }
+if failed { fail 'Idriç decoding producer refused or failed' }
+regular_file "$stage/producer.tsv" 1024
+if ! awk -F b'\t' '
+  BEGIN {
+    split("original_byte_count decoded_codepoint_count selected_encoding decoder_identity", keys, " ")
+    for (i in keys) required[keys[i]]=1
+  }
+  NF!=2 || !($1 in required) || $2=="" || seen[$1]++ || /\r/ {bad=1}
+  END {for (key in required) if (!seen[key]) bad=1; exit bad}
+' "$stage/producer.tsv" { fail 'invalid decoder result fields' }
+if ($(field "$stage/producer.tsv" original_byte_count) !== expected_count or
+    $(field "$stage/producer.tsv" decoded_codepoint_count) !== expected_count) {
+  fail 'decoder count disagrees with frozen source'
+}
+if ($(field "$stage/producer.tsv" selected_encoding) !== 'windows-1252' or
+    $(field "$stage/producer.tsv" decoder_identity) !== 'whatwg-windows-1252-e56d49d9176e9a4-v1') {
+  fail 'unsupported decoder result identity'
+}
+regular_file "$stage/decoded.utf8" $[int(expected_count) * 3]
+if ($(digest "$stage/input.raw") !== expected_hash) { fail 'source changed during decoding' }
+if ($(digest "$stage/source.tsv") !== source_metadata_hash) { fail 'source descriptor changed during decoding' }
+var output_count = $(wc -c < "$stage/decoded.utf8" | tr -d '[:space:]')
+var output_hash = $(digest "$stage/decoded.utf8")
+{
+  printf 'schema\tib-decoded-html-v1\n'
+  printf 'source_key\t%s\nsource_generation\t%s\n' $key $generation
+  printf 'source_sha256\t%s\nsource_byte_count\t%s\n' $expected_hash $expected_count
+  printf 'source_descriptor_sha256\t%s\n' $source_metadata_hash
+  printf 'declared_charset\t%s\nselected_encoding\twindows-1252\n' $charset
+  printf 'decoder_identity\twhatwg-windows-1252-e56d49d9176e9a4-v1\n'
+  printf 'decoded_charset\tutf-8\ndecoded_byte_count\t%s\n' $output_count
+  printf 'decoded_codepoint_count\t%s\ndecoded_sha256\t%s\n' $expected_count $output_hash
+  printf 'original_navigation_admitted\t%s\nreader_admission\tNOT_GRANTED\n' $navigation
+} > "$stage/decode.tsv"
+digest "$stage/decode.tsv" > "$stage/complete"
+rm -- "$stage/input.raw" "$stage/producer.tsv"
+mv -T --no-clobber -- $stage $destination
+if test -d $stage { fail 'derivative publication conflicted' }
+write -- $destination