From 4250efda6544465ee17e1aad960abd63f7f69b77 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:27:25 -0400 Subject: [PATCH 01/55] Preserve six HTML heading levels and decode bounded entities without injecting inline spaces --- src/IB/Information.idric | 86 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/src/IB/Information.idric b/src/IB/Information.idric index ee53bef5..19befbc8 100644 --- a/src/IB/Information.idric +++ b/src/IB/Information.idric @@ -5,6 +5,7 @@ import Data.Text public export data InformationItem = Heading Text + | HeadingAt Text Text | TextBlock Text | Link Text Text | Image Text Text Text Text @@ -67,10 +68,83 @@ collapse_space_chars previous_space (value :: rest) = else ' ' :: collapse_space_chars True rest else value :: collapse_space_chars False rest +-- Decode only explicit, bounded HTML character references. Unknown names +-- and malformed numerical references remain visible, not fabricated Unicode. +entity_digit : Char → Maybe Integer +entity_digit digit = + let value : Integer = cast (ord digit) in + if value >= 48 && value <= 57 then Just (value - 48) + else if value >= 65 && value <= 70 then Just (value - 55) + else if value >= 97 && value <= 102 then Just (value - 87) + else Nothing + +entity_number : Integer → Integer → List Char → Maybe Integer +entity_number base number [] = + if number > 0 && number <= 1114111 && + not (number >= 55296 && number <= 57343) + then Just number else Nothing +entity_number base number (digit :: rest) = + case entity_digit digit of + Nothing ⇒ Nothing + Just value ⇒ + if value >= base || number > 1114111 + then Nothing + else entity_number base (number * base + value) rest + +named_character : Text → Maybe Char +named_character "amp" = Just '&' +named_character "lt" = Just '<' +named_character "gt" = Just '>' +named_character "quot" = Just '"' +named_character "apos" = Just '\'' +named_character "nbsp" = Just ' ' +named_character "copy" = Just '©' +named_character "reg" = Just '®' +named_character "ndash" = Just '–' +named_character "mdash" = Just '—' +named_character "hellip" = Just '…' +named_character "bull" = Just '•' +named_character "times" = Just '×' +named_character "minus" = Just '−' +named_character "alpha" = Just 'α' +named_character "beta" = Just 'β' +named_character "pi" = Just 'π' +named_character _ = Nothing + +entity_character : Text → Maybe Char +entity_character name = + case unpack name of + '#' :: 'x' :: digits ⇒ map (chr . cast) (entity_number 16 0 digits) + '#' :: 'X' :: digits ⇒ map (chr . cast) (entity_number 16 0 digits) + '#' :: digits ⇒ map (chr . cast) (entity_number 10 0 digits) + _ ⇒ named_character name + +scan_entity : Nat → List Char → List Char → Maybe (Text, List Char) +scan_entity Z collected remaining = Nothing +scan_entity (S fuel) collected [] = Nothing +scan_entity (S fuel) collected (';' :: rest) = Just (pack (reverse collected), rest) +scan_entity (S fuel) collected (value :: rest) = + scan_entity fuel (value :: collected) rest + +decode_entity_chars : List Char → List Char +decode_entity_chars [] = [] +decode_entity_chars ('&' :: rest) = + case scan_entity 16 [] rest of + Nothing ⇒ '&' :: decode_entity_chars rest + Just (name, remaining) ⇒ + case entity_character name of + Nothing ⇒ '&' :: decode_entity_chars rest + Just value ⇒ value :: decode_entity_chars remaining +decode_entity_chars (value :: rest) = value :: decode_entity_chars rest + +public export +decode_html_entities : Text → Text +decode_html_entities value = pack (decode_entity_chars (unpack value)) + public export clean_text : Text → Text clean_text text = - let collapsed = collapse_space_chars False (unpack text) in + let collapsed = collapse_space_chars False (decode_entity_chars (unpack text)) in let left_trimmed = drop_space_chars collapsed in pack (reverse (drop_space_chars (reverse left_trimmed))) @@ -265,7 +339,7 @@ attribute_or_empty : Text → Text → Text attribute_or_empty name tag = case attribute_value name tag of Nothing ⇒ "" - Just value ⇒ value + Just value ⇒ decode_html_entities value skipped_tag : Text → Bool skipped_tag name = elem name ["script", "style", "svg", "canvas", "video", "audio", "template"] @@ -283,14 +357,14 @@ capture_target (CaptureForm _) = "form" capture_target (CaptureCell name) = name append_capture_text : Text → List Char → List Char -append_capture_text text chars = reverse (unpack text) ++ (' ' :: chars) +append_capture_text text chars = reverse (unpack text) ++ chars finish_capture : Capture → Text → Text → List InformationItem → Maybe (List Text) → (Text, List InformationItem, Maybe (List Text)) finish_capture NoCapture text title items row = (title, items, row) finish_capture CaptureTitle text title items row = (text, items, row) -finish_capture (CaptureHeading _) text title items row = - if text ≟ "" then (title, items, row) else (title, Heading text :: items, row) +finish_capture (CaptureHeading name) text title items row = + if text ≟ "" then (title, items, row) else (title, HeadingAt name text :: items, row) finish_capture (CaptureText _) text title items row = if text ≟ "" then (title, items, row) else (title, TextBlock text :: items, row) finish_capture (CaptureLink href has_image parent) text title items row = @@ -508,6 +582,7 @@ join_cells (value :: rest) = value ++ " " ++ join_cells rest item_text : InformationItem → Text item_text (Heading text) = text +item_text (HeadingAt name text) = text item_text (TextBlock text) = text item_text (Link text href) = text ++ " " ++ href item_text (Image source alt caption href) = alt ++ " " ++ source ++ " " ++ caption ++ " " ++ href @@ -542,6 +617,7 @@ join_with separator (value :: rest) = value ++ separator ++ join_with separator render_item : Number → InformationItem → Text render_item width (Heading text) = clip_text width ("# " ++ text) +render_item width (HeadingAt name text) = clip_text width ("# " ++ text) render_item width (TextBlock text) = clip_text width text render_item width (Link text href) = clip_text width ("link: " ++ text ++ " -> " ++ href) render_item width (Image source alt caption href) = From 57bcf83c776bbcc3762bd04724886db1cac96e00 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:27:58 -0400 Subject: [PATCH 02/55] Add source-base-aware HTTP reference resolution for admitted HTML --- src/IB/HtmlLinks.idric | 139 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/IB/HtmlLinks.idric diff --git a/src/IB/HtmlLinks.idric b/src/IB/HtmlLinks.idric new file mode 100644 index 00000000..2109fa15 --- /dev/null +++ b/src/IB/HtmlLinks.idric @@ -0,0 +1,139 @@ +module IB.HtmlLinks + +import IB.Information + +%default total + +-- This is a deliberately bounded address resolver for admitted HTML links. +-- It does not decode percent escapes or extend ICU's supported URL schemes. +strip_prefix : List Char → List Char → Maybe (List Char) +strip_prefix [] rest = Just rest +strip_prefix (_ :: _) [] = Nothing +strip_prefix (wanted :: following) (actual :: rest) = + if wanted == actual then strip_prefix following rest else Nothing + +starts : String → String → Bool +starts prefix value = + case strip_prefix (unpack prefix) (unpack value) of + Nothing ⇒ False + Just _ ⇒ True + +before_character : Char → List Char → List Char +before_character delimiter [] = [] +before_character delimiter (value :: rest) = + if value == delimiter then [] else value :: before_character delimiter rest + +before_either : List Char → List Char +before_either [] = [] +before_either (value :: rest) = + if value == '/' || value == '?' || value == '#' + then [] else value :: before_either rest + +after_count : Nat → List Char → List Char +after_count Z characters = characters +after_count (S count) [] = [] +after_count (S count) (_ :: rest) = after_count count rest + +public export +http_origin : String → Maybe String +http_origin url = + if not (cheap_http_url url) + then Nothing + else if starts "https://" url + then Just ("https://" ++ pack (before_either (after_count 8 (unpack url)))) + else if starts "http://" url + then Just ("http://" ++ pack (before_either (after_count 7 (unpack url)))) + else Nothing + +url_tail : String → String → String +url_tail origin url = pack (after_count (length (unpack origin)) (unpack url)) + +strip_fragment : String → String +strip_fragment value = pack (before_character '#' (unpack value)) + +strip_query : String → String +strip_query value = pack (before_character '?' (unpack value)) + +url_path : String → String → String +url_path origin url = + let rest = url_tail origin url in + case unpack rest of + '/' :: _ ⇒ strip_query (strip_fragment rest) + _ ⇒ "/" + +parent_path : List Char → List Char → List Char +parent_path [] parent = reverse parent +parent_path ('/' :: rest) previous = parent_path rest ('/' :: previous) +parent_path (value :: rest) previous = + case rest of + [] ⇒ reverse previous + _ ⇒ parent_path rest (value :: previous) + +-- Segment operations keep the origin outside the path stack. +segments : List Char → List Char → List String → List String +segments [] current done = reverse (pack (reverse current) :: done) +segments ('/' :: rest) current done = + segments rest [] (pack (reverse current) :: done) +segments (value :: rest) current done = segments rest (value :: current) done + +normalise : List String → List String → List String +normalise [] stack = reverse stack +normalise ("." :: rest) stack = normalise rest stack +normalise (".." :: rest) [] = normalise rest [] +normalise (".." :: rest) (_ :: stack) = normalise rest stack +normalise (value :: rest) stack = normalise rest (value :: stack) + +join_segments : List String → String +join_segments [] = "" +join_segments [value] = value +join_segments (value :: rest) = value ++ "/" ++ join_segments rest + +split_path_suffix : List Char → List Char → (String, String) +split_path_suffix [] path = (pack (reverse path), "") +split_path_suffix chars@('?' :: _) path = (pack (reverse path), pack chars) +split_path_suffix chars@('#' :: _) path = (pack (reverse path), pack chars) +split_path_suffix (value :: rest) path = split_path_suffix rest (value :: path) + +path_normalised : String → String +path_normalised path = + case unpack path of + '/' :: rest ⇒ "/" ++ join_segments (normalise (segments rest [] []) []) + rest ⇒ "/" ++ join_segments (normalise (segments rest [] []) []) + +has_scheme_colon : List Char → Bool +has_scheme_colon [] = False +has_scheme_colon ('/' :: _) = False +has_scheme_colon ('?' :: _) = False +has_scheme_colon ('#' :: _) = False +has_scheme_colon (':' :: _) = True +has_scheme_colon (_ :: rest) = has_scheme_colon rest + +-- Returns a navigation *candidate*. ICU and navigation admission still decide +-- whether another retrieval occurs; the renderer must never fetch a link. +public export +resolve_html_reference : String → String → Maybe String +resolve_html_reference base reference = + if not (cheap_http_url base) || reference == "" + then Nothing + else if cheap_http_url reference + then Just reference + else if starts "//" reference + then + let candidate = (if starts "https://" base then "https:" else "http:") ++ reference in + if cheap_http_url candidate then Just candidate else Nothing + else if has_scheme_colon (unpack reference) + then Nothing + else do + origin ← http_origin base + let current = strip_fragment base + let candidate = + if starts "#" reference then current ++ reference + else if starts "?" reference + then strip_query current ++ reference + else + let (relative_path, suffix) = split_path_suffix (unpack reference) [] in + let prefix = + if starts "/" reference then "" + else pack (parent_path (unpack (url_path origin base)) []) in + origin ++ path_normalised (prefix ++ relative_path) ++ suffix + if cheap_http_url candidate then Just candidate else Nothing From a4e6a9163ad968d6182b98678f369c0c22b733c0 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:28:51 -0400 Subject: [PATCH 03/55] Convert admitted recovered HTML to typed Strand and actual Prepaint Version 1 --- src/IB/HtmlPrepaint.idric | 222 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 src/IB/HtmlPrepaint.idric diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric new file mode 100644 index 00000000..b407db07 --- /dev/null +++ b/src/IB/HtmlPrepaint.idric @@ -0,0 +1,222 @@ +module IB.HtmlPrepaint + +import IB.Information +import IB.RecoveredInformation +import IB.Strand +import IB.Prepaint +import IB.Prepaint.Version1 +import IB.HtmlLinks + +%default total + +-- A producer supplies this ONLY after checking the exact committed Arena range. +-- Claimed and reopened digest/length are independent inputs, not a hash of an +-- unverified String. Source admission and byte-level SHA-256 belong to Grease / +-- the canonical Arena reader; semantic extraction belongs here. +public export +record HtmlEvidence where + constructor HtmlSourceEvidence + source_key : String + source_generation : String + claimed_sha256 : String + reopened_sha256 : String + claimed_bytes : Integer + reopened_bytes : Integer + requested_url : String + resolved_url : String + mime_type : String + charset : String + http_status : String + utf8_decoded : Bool + fully_acquired : Bool + +public export +record PendingImage where + constructor NeedImageAcquisition + image_url : String + alternate : String + caption : String + source_key : String + +public export +record HtmlPrepaintResult where + constructor MadeHtmlPrepaint + evidence : HtmlEvidence + recovered : RecoveredDocumentLog + information : InformationView + strand : Strand + revision : PrepaintRevision + wire : String + pending_images : List PendingImage + limitations : List String + +is_hex_character : Char → Bool +is_hex_character character = + let number : Integer = cast (ord character) in + (number >= 48 && number <= 57) || (number >= 97 && number <= 102) + +is_digest : String → Bool +is_digest value = + length (unpack value) == 64 && all is_hex_character (unpack value) + +metadata_safe : String → Bool +metadata_safe value = + value /= "" && length (unpack value) <= 256 && + not (elem '\t' (unpack value)) && + not (elem '\n' (unpack value)) && + not (elem '\r' (unpack value)) + +public export +source_admission_reason : HtmlEvidence → Maybe String +source_admission_reason e = + if not (metadata_safe e.source_key) || not (metadata_safe e.source_generation) + then Just "bad-source-identity" + else if not (is_digest e.claimed_sha256) || e.claimed_sha256 /= e.reopened_sha256 + then Just "missing-or-mismatched-sha256" + else if e.claimed_bytes <= 0 || e.claimed_bytes > 4194304 || + e.claimed_bytes /= e.reopened_bytes + then Just "bad-reopened-byte-count" + else if e.mime_type /= "text/html" && + e.mime_type /= "text/html; charset=utf-8" && + e.mime_type /= "text/html; charset=UTF-8" + then Just "unsupported-mime" + else if e.charset /= "utf-8" && e.charset /= "UTF-8" + then Just "unsupported-charset" + else if e.http_status /= "200" + then Just "unadmitted-http-status" + else if not e.utf8_decoded + then Just "invalid-utf8" + else if not e.fully_acquired + then Just "incomplete-or-stale-source" + else if not (cheap_http_url e.requested_url) || not (cheap_http_url e.resolved_url) + then Just "unverified-requested-or-resolved-url" + else Nothing + +first_html_base : List DocumentEvent → Maybe String +first_html_base [] = Nothing +first_html_base (StartTag "base" tag :: events) = + case attribute_value "href" tag of + Nothing ⇒ first_html_base events + Just value ⇒ Just (decode_html_entities value) +first_html_base (_ :: events) = first_html_base events + +link_base : String → RecoveredDocumentLog → (String, List String) +link_base resolved log = + case first_html_base (document_log_events log) of + Nothing ⇒ (resolved, []) + Just href ⇒ + case resolve_html_reference resolved href of + Nothing ⇒ (resolved, ["unsupported-html-base"]) + Just value ⇒ (value, []) + +heading_number : String → Maybe HeadingLevel +heading_number "h1" = heading_level 1 +heading_number "h2" = heading_level 2 +heading_number "h3" = heading_level 3 +heading_number "h4" = heading_level 4 +heading_number "h5" = heading_level 5 +heading_number "h6" = heading_level 6 +heading_number _ = Nothing + +checked_link : String → String → String → + (List InformationBlock, List String) +checked_link base label href = + case resolve_html_reference base href of + Nothing ⇒ ([IB.Strand.TextBlock label], ["unusable-link:" ++ href]) + Just url ⇒ + case link_reference url of + Nothing ⇒ ([IB.Strand.TextBlock label], ["empty-link"]) + Just ref ⇒ ([IB.Strand.Link label ref], []) + +map_item : HtmlEvidence → String → InformationItem → + (List InformationBlock, List PendingImage, List String) +map_item e base (IB.Information.Heading text) = + ([IB.Strand.Heading Heading1 text], [], ["legacy-heading-level-unknown"]) +map_item e base (HeadingAt name text) = + case heading_number name of + Nothing ⇒ ([IB.Strand.TextBlock text], [], ["invalid-heading-level"]) + Just level ⇒ ([IB.Strand.Heading level text], [], []) +map_item e base (IB.Information.TextBlock text) = + ([IB.Strand.TextBlock text], [], []) +map_item e base (IB.Information.Link text href) = + let (blocks, notes) = checked_link base text href in + (blocks, [], notes) +map_item e base (IB.Information.Form description action) = + ([IB.Strand.TextBlock ("Form (submission unavailable): " ++ description)], + [], ["form-submission-unavailable:" ++ action]) +map_item e base (IB.Information.TableRow []) = + ([], [], ["empty-table-row"]) +map_item e base (IB.Information.TableRow (first :: cells)) = + ([IB.Strand.TableRow (table_row first cells)], [], []) +map_item e base (IB.Information.Image url alt caption destination) = + let text = if alt == "" then "Unretrieved image" else alt in + let label = if caption == "" then text else text ++ " — " ++ caption in + let pending = + case resolve_html_reference base url of + Nothing ⇒ [] + Just source ⇒ [NeedImageAcquisition source alt caption e.source_key] in + let image_notes = + if pending == [] then ["unsupported-image-reference:" ++ url] + else ["image-requires-separate-admission:" ++ url] in + let (linked, link_notes) = + if destination == "" then ([], []) + else checked_link base label destination in + (IB.Strand.TextBlock ("Image pending acquisition: " ++ label) :: linked, + pending, image_notes ++ link_notes) + +map_items : HtmlEvidence → String → List InformationItem → + (List InformationBlock, List PendingImage, List String) +map_items e base [] = ([], [], []) +map_items e base (item :: rest) = + let (block, images, notes) = map_item e base item in + let (other_blocks, other_images, other_notes) = map_items e base rest in + (block ++ other_blocks, images ++ other_images, notes ++ other_notes) + +unsupported_event : DocumentEvent → List String +unsupported_event (StartTag name _) = + if elem name ["script", "style", "template", "canvas", "video", + "audio", "svg", "iframe", "object"] + then ["excluded-or-unsupported-element:" ++ name] else [] +unsupported_event _ = [] + +unsupported_events : List DocumentEvent → List String +unsupported_events [] = [] +unsupported_events (event :: rest) = + unsupported_event event ++ unsupported_events rest + +public export +convert_admitted_html : HtmlEvidence → String → Either String HtmlPrepaintResult +convert_admitted_html admission html = + case source_admission_reason admission of + Just error ⇒ Left error + Nothing ⇒ + let log = recover_document_log admission.source_key html in + let view = information_from_log admission.requested_url + admission.resolved_url log in + let (base, base_notes) = link_base admission.resolved_url log in + let (blocks, images, notes) = map_items admission base (view_items view) in + if length blocks > 4096 + then Left "prepaint-block-budget-exceeded" + else do + source_id ← case nonempty_text + (admission.source_key ++ ";generation=" ++ admission.source_generation ++ + ";sha256=" ++ admission.claimed_sha256) of + Nothing ⇒ Left "empty-source" + Just value ⇒ Right value + requested ← case requested_address admission.requested_url of + Nothing ⇒ Left "empty-requested-address" + Just value ⇒ Right value + resolved ← case resolved_address admission.resolved_url of + Nothing ⇒ Left "empty-resolved-address" + Just value ⇒ Right value + let source = Source source_id HtmlSource CompleteSource + let title = + if view_title view == "" then Nothing else Just (view_title view) + let strand = MakeStrand source requested resolved title blocks + let revision = prepaint_from_strand first_revision CompleteRevision strand + case serialize_version1 [revision] of + Nothing ⇒ Left "version1-serialization-refused" + Just wire ⇒ + Right (MadeHtmlPrepaint admission log view strand revision wire + images (base_notes ++ notes ++ + unsupported_events (document_log_events log))) From de3f7f68a0df959f058cd321a350026657fd3232 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:29:45 -0400 Subject: [PATCH 04/55] =?UTF-8?q?Add=20executable=20Idri=C3=A7=20HTML=20pr?= =?UTF-8?q?epaint=20command=20and=20converter=20semantic=20smoke?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/HtmlPrepaintSmoke.idric | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/HtmlPrepaintSmoke.idric diff --git a/src/HtmlPrepaintSmoke.idric b/src/HtmlPrepaintSmoke.idric new file mode 100644 index 00000000..cfdb420b --- /dev/null +++ b/src/HtmlPrepaintSmoke.idric @@ -0,0 +1,91 @@ +module HtmlPrepaintSmoke + +import IB.HtmlPrepaint +import IB.HtmlLinks +import IB.Strand +import System + +source_sha : String +source_sha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +example : String +example = + "A & B" ++ + "

First < example

Sixth
" ++ + "

beforeboldafter

" ++ + "

prefix link suffix

" ++ + "

smile: 🙂 RTL: שלום é

" ++ + "
AB
" ++ + "Unsafe" ++ + "diagram" ++ + "" + +evidence : HtmlEvidence +evidence = + HtmlSourceEvidence "arena:fixture-example" "generation-1" + source_sha source_sha 500 500 + "https://example.com/docs/page" "https://example.com/docs/page" + "text/html" "utf-8" "200" True True + +bad_evidence : HtmlEvidence +bad_evidence = + HtmlSourceEvidence "arena:fixture-example" "generation-1" + source_sha "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + 500 500 "https://example.com/" "https://example.com/" + "text/html" "utf-8" "200" True True + +contains : List Char → List Char → Bool +contains needle [] = needle == [] +contains needle value@(_ :: rest) = + starts_with needle value || contains needle rest + where + starts_with : List Char → List Char → Bool + starts_with [] _ = True + starts_with (_ :: _) [] = False + starts_with (x :: xs) (y :: ys) = + x == y && starts_with xs ys + +has : String → String → Bool +has sub value = contains (unpack sub) (unpack value) + +refused : Either String HtmlPrepaintResult → Bool +refused (Left _) = True +refused (Right _) = False + +check : List (String, Bool) → IO Bool +check [] = pure True +check ((name, passed) :: rest) = do + putStrLn (name ++ "=" ++ show passed) + more <- check rest + pure (passed && more) + +main : IO () +main = + case convert_admitted_html evidence example of + Left reason ⇒ die ("html-prepaint=" ++ reason) + Right output ⇒ do + let wire = output.wire + let tests = + [ ("version1", has "ib-prepaint\t1\nrevision\t0\tcomplete\n" wire) + , ("heading-one", has "heading\t1\tFirst < example\n" wire) + , ("heading-six", has "heading\t6\tSixth\n" wire) + , ("nested-inline", has "text\tbeforeboldafter\n" wire) + , ("relative-link", has "link\tlink\thttps://example.com/other?q=1\n" wire) + , ("paragraph-before-link", has "text\tprefix\n" wire) + , ("paragraph-after-link", has "text\tsuffix\n" wire) + , ("non-bmp", has "🙂" wire) + , ("numeric-combining", has "é" wire) + , ("table-row", has "row\tA\tB\n" wire) + , ("unsafe-no-link", not (has "link\tUnsafe" wire)) + , ("image-unretrieved", has "Image pending acquisition: diagram — caption" wire) + , ("script-excluded", not (has "hidden" wire)) + , ("digest-mismatch-refused", refused (convert_admitted_html bad_evidence example)) + , ("fragment", resolve_html_reference "https://example.com/a?q=1#old" "#new" == + Just "https://example.com/a?q=1#new") + , ("query", resolve_html_reference "https://example.com/a?q=1" "?q=2" == + Just "https://example.com/a?q=2") + , ("unsafe-scheme", resolve_html_reference "https://example.com/" "javascript:bad" == + Nothing) + ] + okay <- check tests + if okay then pure () else exitFailure From 26a2ef6a1f92b5046ccd67722cd7ab1be24aec49 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:29:47 -0400 Subject: [PATCH 05/55] =?UTF-8?q?Add=20executable=20Idri=C3=A7=20HTML=20pr?= =?UTF-8?q?epaint=20command=20and=20converter=20semantic=20smoke?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/HtmlPrepaintCommand.idric | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/HtmlPrepaintCommand.idric diff --git a/src/HtmlPrepaintCommand.idric b/src/HtmlPrepaintCommand.idric new file mode 100644 index 00000000..f6e0d030 --- /dev/null +++ b/src/HtmlPrepaintCommand.idric @@ -0,0 +1,33 @@ +module HtmlPrepaintCommand + +import IB.HtmlPrepaint +import System +import System.File + +-- The caller must independently reopen/hash the *exact* Arena range before +-- constructing these arguments. This executable does no I/O except one +-- bounded already-verified UTF-8 body read and writing the disposable wire. +program_arguments : List String → List String +program_arguments [] = [] +program_arguments (_ :: args) = args + +covering +main : IO () +main = do + arguments <- getArgs + case program_arguments arguments of + [source_id, generation, claimed_sha, reopened_sha, + claimed_bytes, reopened_bytes, requested, resolved, + mime, charset, status, body_path] ⇒ do + data_result <- readFile body_path + case data_result of + Left _ ⇒ die "html-prepaint: admitted body unreadable" + Right html ⇒ do + let evidence = + HtmlSourceEvidence source_id generation claimed_sha reopened_sha + (cast claimed_bytes) (cast reopened_bytes) + requested resolved mime charset status True True + case convert_admitted_html evidence html of + Left reason ⇒ die ("html-prepaint: " ++ reason) + Right prepaint ⇒ putStr prepaint.wire + _ ⇒ die "usage: ib-html-prepaint SOURCE GENERATION CLAIM_SHA REOPEN_SHA CLAIM_COUNT REOPEN_COUNT REQUESTED FINAL MIME CHARSET STATUS ADMITTED_BODY_PATH" From 092de1402bcefec0fabef309bf01ff23fe44b106 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:30:15 -0400 Subject: [PATCH 06/55] Avoid requiring equality on pending image acquisition requests --- src/IB/HtmlPrepaint.idric | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index b407db07..99964578 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -156,8 +156,9 @@ map_item e base (IB.Information.Image url alt caption destination) = Nothing ⇒ [] Just source ⇒ [NeedImageAcquisition source alt caption e.source_key] in let image_notes = - if pending == [] then ["unsupported-image-reference:" ++ url] - else ["image-requires-separate-admission:" ++ url] in + case pending of + [] ⇒ ["unsupported-image-reference:" ++ url] + _ ⇒ ["image-requires-separate-admission:" ++ url] in let (linked, link_notes) = if destination == "" then ([], []) else checked_link base label destination in From 3e7702d81f51fb24db7cca460cd30cffd78017af Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:30:24 -0400 Subject: [PATCH 07/55] =?UTF-8?q?Compile=20and=20run=20actual=20HTML-to-V1?= =?UTF-8?q?=20Idri=C3=A7=20semantic=20smoke=20in=20existing=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/idric-core.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 5e802820..765e04a7 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -101,6 +101,18 @@ jobs: grep -Fx 'nonincreasing-revision-refused=True' /tmp/ib-canonical-information.txt grep -Fx 'version1=serialized' /tmp/ib-canonical-information.txt + - name: Compile and exercise admitted HTML to Prepaint Version 1 + working-directory: src + run: | + /tmp/idric/bin/idris2 HtmlPrepaintSmoke.idric -o ib-html-prepaint-smoke + test -x build/exec/ib-html-prepaint-smoke + build/exec/ib-html-prepaint-smoke | tee /tmp/html-prepaint-smoke.txt + grep -Fx 'version1=True' /tmp/html-prepaint-smoke.txt + grep -Fx 'heading-six=True' /tmp/html-prepaint-smoke.txt + grep -Fx 'digest-mismatch-refused=True' /tmp/html-prepaint-smoke.txt + /tmp/idric/bin/idris2 HtmlPrepaintCommand.idric -o ib-html-prepaint + test -x build/exec/ib-html-prepaint + - name: Exercise browser-owned Idriç core run: sh bin/ci_browser_foundation.grease exercise-core From 3c2d6dbd53124a1d8ec06a17b24b4704240d1674 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:31:49 -0400 Subject: [PATCH 08/55] Add fail-closed Grease admission and disposable prepaint publication --- bin/ib_emit_html_prepaint.grease | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 bin/ib_emit_html_prepaint.grease diff --git a/bin/ib_emit_html_prepaint.grease b/bin/ib_emit_html_prepaint.grease new file mode 100644 index 00000000..fe3a371c --- /dev/null +++ b/bin/ib_emit_html_prepaint.grease @@ -0,0 +1,65 @@ +#!/bin/sh +# Grease adapter for independently reopened and admitted HTML. +# Does not acquire sources, network, or create a canonical store. +set -eu +umask 077 +fail() { printf 'ib html prepaint: %s\n' "$1" >&2; exit 1; } +test "$#" -eq 4 || fail 'usage: ib_emit_html_prepaint.grease ADMISSION.tsv VERIFIED_BODY EXPECTED_SOURCE_KEY OUTPUT' +manifest=$1 +body=$2 +expected_key=$3 +output=$4 +test -f "$manifest" && test ! -L "$manifest" || fail 'missing or linked admission' +test -f "$body" && test ! -L "$body" || fail 'missing or linked admitted body' +test ! -L "$output" || fail 'refusing output symlink' +test "$(wc -c < "$manifest" | tr -d '[:space:]')" -le 4096 || fail 'oversized admission' +awk -F '\t' ' + NF != 2 || $1 !~ /^[a-z_]+$/ || $2 == "" || /\r/ { bad=1 } + { if (seen[$1]++) bad=1 } + END { if (bad || NR != 13) exit 1 } +' "$manifest" || fail 'bad admission structure' +field() { awk -F '\t' -v key="$1" '$1 == key { print $2 }' "$manifest"; } +test "$(field schema)" = ib-html-admission-v1 || fail 'unsupported admission schema' +test "$(field source_key)" = "$expected_key" || fail 'stale or modified source key' +case "$(field source_kind)" in + arena) test "$(field arena_reopen)" = MATCH || fail 'Arena readback absent' ;; + fixture) test "$(printenv IB_ALLOW_FIXTURE 2>/dev/null || :)" = 1 || fail 'fixture not allowed as production source' ;; + *) fail 'unknown source kind' ;; +esac +test "$(field acquisition_complete)" = YES || fail 'source incomplete' +test "$(field http_status)" = 200 || fail 'unadmitted HTTP status' +test "$(field mime_type)" = text/html || fail 'not admitted HTML' +case "$(field charset)" in utf-8|UTF-8) ;; *) fail 'unsupported charset' ;; esac +test "$(field source_generation)" != UNOBSERVED || fail 'generation not observed' +test "$(field resolved_url)" != UNOBSERVED || fail 'final URL not observed' +test "$(field requested_url)" != UNOBSERVED || fail 'requested URL not observed' +claimed=$(field sha256) +bytes=$(field byte_count) +case "$claimed" in *[!0123456789abcdef]*|'') fail 'invalid digest' ;; esac +test "$(printf %s "$claimed" | wc -c | tr -d '[:space:]')" -eq 64 || fail 'invalid digest length' +case "$bytes" in *[!0123456789]*|'') fail 'invalid byte count' ;; esac +test "$bytes" -gt 0 && test "$bytes" -le 4194304 || fail 'body exceeds admission bound' +observed_bytes=$(wc -c < "$body" | tr -d '[:space:]') +test "$bytes" = "$observed_bytes" || fail 'body length differs from committed source' +observed=$(sha256sum "$body" | cut -d ' ' -f 1) +test "$claimed" = "$observed" || fail 'body digest differs from committed source' +iconv -f UTF-8 -t UTF-8 "$body" >/dev/null || fail 'invalid UTF-8' +converter=$(printenv IB_HTML_PREPAINT_EXEC 2>/dev/null || :) +oracle=$(printenv IB_PREPAINT_ORACLE 2>/dev/null || :) +test -n "$converter" && test -x "$converter" || fail 'compiled Idriç executable required' +test -n "$oracle" && test -x "$oracle" || fail 'independent Version 1 consumer required' +parent=$(dirname -- "$output") +test -d "$parent" && test ! -L "$parent" || fail 'disposable output directory absent' +work=$(mktemp -d "$parent/.ib-prepaint.XXXXXXXX") || fail 'cannot stage disposable output' +cleanup() { rm -rf "$work"; } +trap cleanup EXIT HUP INT TERM +"$converter" "$(field source_key)" "$(field source_generation)" "$claimed" "$observed" \ + "$bytes" "$observed_bytes" "$(field requested_url)" "$(field resolved_url)" \ + "$(field mime_type)" "$(field charset)" "$(field http_status)" "$body" \ + > "$work/projection" || fail 'Idriç converter refused admitted HTML' +test "$(wc -c < "$work/projection" | tr -d '[:space:]')" -le 4194304 || fail 'prepaint exceeds display bound' +"$oracle" "$work/projection" || fail 'independent Version 1 oracle refused output' +test ! -L "$output" || fail 'output symlink changed' +mv -f "$work/projection" "$output" || fail 'projection publication failed' +printf 'conversion\tPASS\nsource_key\t%s\nsource_generation\t%s\nsource_sha256\t%s\nprepaint_sha256\t%s\n' \ + "$expected_key" "$(field source_generation)" "$observed" "$(sha256sum "$output" | cut -d ' ' -f 1)" From 8a7d332b41100182fe663d2b0e19c9571b76663c Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:32:25 -0400 Subject: [PATCH 09/55] Add independent bounded C11 consumer for actual complete ib-prepaint Version 1 --- tests/prepaint_v1_oracle.c | 177 +++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 tests/prepaint_v1_oracle.c diff --git a/tests/prepaint_v1_oracle.c b/tests/prepaint_v1_oracle.c new file mode 100644 index 00000000..ccf0e4f3 --- /dev/null +++ b/tests/prepaint_v1_oracle.c @@ -0,0 +1,177 @@ +/* + * Independent narrow Version 1 *consumer*, not a renderer or alternate HTML + * parser. The producer/Idriç serializer is not linked into this executable. + * It validates the published complete projection's grammar and unsafe targets. + */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include + +enum { LIMIT_BYTES = 4194304, LIMIT_BLOCKS = 4096, LIMIT_REVISIONS = 32 }; +static int starts(const char *value, const char *prefix) { + return strncmp(value, prefix, strlen(prefix)) == 0; +} +static int http_url(const char *value) { + const char *host; + if (starts(value, "https://")) host = value + 8; + else if (starts(value, "http://")) host = value + 7; + else return 0; + if (!*host || *host == '/' || *host == '?' || *host == '#') return 0; + for (const char *p = host; *p && *p != '/' && *p != '?' && *p != '#'; ++p) { + if ((unsigned char)*p <= 32 || (unsigned char)*p >= 127 || *p == '@') return 0; + } + for (const unsigned char *p = (const unsigned char *)value; *p; ++p) + if (*p <= 32 || *p >= 127) return 0; + return 1; +} +static int local_image(const char *value) { + return starts(value, "content:") || starts(value, "asset:") || + starts(value, "resource:") || starts(value, "file:"); +} +/* Reject malformed UTF-8, overlong sequences, surrogates and >U+10FFFF. */ +static int valid_utf8(const unsigned char *p, size_t n) { + for (size_t i = 0; i < n;) { + unsigned c = p[i++]; + if (c < 0x80) { if (c == 0) return 0; continue; } + unsigned need, min; + if (c >= 0xc2 && c <= 0xdf) { need = 1; min = 0x80; c &= 0x1f; } + else if (c >= 0xe0 && c <= 0xef) { need = 2; min = 0x800; c &= 0x0f; } + else if (c >= 0xf0 && c <= 0xf4) { need = 3; min = 0x10000; c &= 0x07; } + else return 0; + if (i + need > n) return 0; + for (unsigned j = 0; j < need; ++j) { + unsigned d = p[i++]; + if ((d & 0xc0) != 0x80) return 0; + c = (c << 6) | (d & 0x3f); + } + if (c < min || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) return 0; + } + return 1; +} +static int number(const char *value, unsigned long *out) { + if (!*value) return 0; + unsigned long x = 0; + for (const unsigned char *p = (const unsigned char *)value; *p; ++p) { + if (*p < '0' || *p > '9' || x > (ULONG_MAX - (*p - '0')) / 10) return 0; + x = x * 10 + *p - '0'; + } + *out = x; + return 1; +} +/* Every tab is a structural delimiter; literal tabs in values must be \\t. */ +static int fields(char *line, char **parts, size_t *count) { + size_t n = 0; + parts[n++] = line; + for (char *p = line; *p; ++p) { + if (*p == '\r' || *p == '\n') return 0; + if (*p == '\\') { + if (!p[1] || !(p[1] == 'n' || p[1] == 'r' || + p[1] == 't' || p[1] == '\\')) return 0; + ++p; + } else if (*p == '\t') { + if (n >= LIMIT_BLOCKS + 5) return 0; + *p = 0; + parts[n++] = p + 1; + } + } + *count = n; + return 1; +} +static int check_file(const char *filename) { + FILE *f = fopen(filename, "rb"); + if (!f) return 0; + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; } + long length = ftell(f); + if (length < 0 || length > LIMIT_BYTES || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); return 0; + } + size_t n = (size_t)length; + char *buf = malloc(n + 1); + char **parts = malloc((LIMIT_BLOCKS + 5) * sizeof *parts); + if (!buf || !parts) { free(buf); free(parts); fclose(f); return 0; } + int ok = fread(buf, 1, n, f) == n && !ferror(f); + fclose(f); + if (!ok || !valid_utf8((const unsigned char *)buf, n)) { + free(buf); free(parts); return 0; + } + buf[n] = 0; + unsigned long previous = 0; + unsigned revisions = 0, blocks = 0; + /* state 0=header, 1=revision, 2=requested, 3=resolved, + * 4=title, 5=blocks/end */ + int state = 0; + size_t cursor = 0; + for (; cursor < n && ok;) { + char *line = buf + cursor; + char *end = memchr(line, '\n', n - cursor); + if (!end) { ok = 0; break; } + *end = 0; + cursor = (size_t)(end - buf) + 1; + size_t fields_count = 0; + if (!fields(line, parts, &fields_count)) { ok = 0; break; } + const char *kind = parts[0]; + if (state == 0) { + ok = fields_count == 2 && strcmp(kind, "ib-prepaint") == 0 && + strcmp(parts[1], "1") == 0; + state = 1; + } else if (state == 1) { + unsigned long seq = 0; + ok = fields_count == 3 && strcmp(kind, "revision") == 0 && + number(parts[1], &seq) && (revisions == 0 || seq > previous) && + strcmp(parts[2], "complete") == 0 && + ++revisions <= LIMIT_REVISIONS; + previous = seq; blocks = 0; state = 2; + } else if (state == 2) { + ok = fields_count == 2 && strcmp(kind, "requested-url") == 0 && + http_url(parts[1]); state = 3; + } else if (state == 3) { + ok = fields_count == 2 && strcmp(kind, "resolved-url") == 0 && + http_url(parts[1]); state = 4; + } else if (state == 4) { + ok = fields_count == 2 && strcmp(kind, "title") == 0; + state = 5; + } else if (strcmp(kind, "end") == 0) { + ok = fields_count == 1; + state = 1; + } else { + ++blocks; + ok = blocks <= LIMIT_BLOCKS; + if (strcmp(kind, "heading") == 0) { + ok = ok && fields_count == 3 && strlen(parts[1]) == 1 && + parts[1][0] >= '1' && parts[1][0] <= '6'; + } else if (strcmp(kind, "text") == 0) { + ok = ok && fields_count == 2; + } else if (strcmp(kind, "link") == 0) { + ok = ok && fields_count == 3 && http_url(parts[2]); + } else if (strcmp(kind, "row") == 0) { + ok = ok && fields_count >= 2; + } else if (strcmp(kind, "form") == 0) { + /* The current converter emits non-interactive form descriptions. */ + ok = 0; + } else if (strcmp(kind, "image") == 0) { + ok = ok && (fields_count == 4 || fields_count == 5) && + local_image(parts[1]) && + (fields_count != 5 || http_url(parts[4])); + } else ok = 0; + } + } + ok = ok && revisions > 0 && state == 1 && cursor == n; + free(buf); + free(parts); + return ok; +} +int main(int argc, char **argv) { + if (argc != 2) { + fprintf(stderr, "usage: prepaint-v1-oracle FILE\n"); + return 2; + } + if (!check_file(argv[1])) { + fprintf(stderr, "independent-prepaint-v1: REFUSED\n"); + return 1; + } + puts("independent-prepaint-v1: PASS"); + return 0; +} From 9860d430ef7e7750be250e95a9a904e32f184bcf Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:32:53 -0400 Subject: [PATCH 10/55] Require caller-pinned source generation before overwriting last valid prepaint --- bin/ib_emit_html_prepaint.grease | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/ib_emit_html_prepaint.grease b/bin/ib_emit_html_prepaint.grease index fe3a371c..6a0dd64c 100644 --- a/bin/ib_emit_html_prepaint.grease +++ b/bin/ib_emit_html_prepaint.grease @@ -21,6 +21,8 @@ awk -F '\t' ' field() { awk -F '\t' -v key="$1" '$1 == key { print $2 }' "$manifest"; } test "$(field schema)" = ib-html-admission-v1 || fail 'unsupported admission schema' test "$(field source_key)" = "$expected_key" || fail 'stale or modified source key' +expected_generation=$(printenv IB_EXPECTED_SOURCE_GENERATION 2>/dev/null || :) +test -n "$expected_generation" && test "$(field source_generation)" = "$expected_generation" || fail 'stale or unpinned generation' case "$(field source_kind)" in arena) test "$(field arena_reopen)" = MATCH || fail 'Arena readback absent' ;; fixture) test "$(printenv IB_ALLOW_FIXTURE 2>/dev/null || :)" = 1 || fail 'fixture not allowed as production source' ;; From dbed934469a82f5d16774944002baa757081f30a Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:33:01 -0400 Subject: [PATCH 11/55] Add offline HTML and independent expected Prepaint Version 1 fixtures --- tests/fixtures/ib-html/example.html | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/fixtures/ib-html/example.html diff --git a/tests/fixtures/ib-html/example.html b/tests/fixtures/ib-html/example.html new file mode 100644 index 00000000..e8770261 --- /dev/null +++ b/tests/fixtures/ib-html/example.html @@ -0,0 +1,6 @@ + +Example Domain +

Example Domain

+

This domain is for use in illustrative examples.

+

Learn more

+ From 82a694b86de69d5f07dcf084038135882f61eada Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:33:04 -0400 Subject: [PATCH 12/55] Add offline HTML and independent expected Prepaint Version 1 fixtures --- tests/fixtures/ib-html/example.prepaint | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/fixtures/ib-html/example.prepaint diff --git a/tests/fixtures/ib-html/example.prepaint b/tests/fixtures/ib-html/example.prepaint new file mode 100644 index 00000000..abdabec9 --- /dev/null +++ b/tests/fixtures/ib-html/example.prepaint @@ -0,0 +1,9 @@ +ib-prepaint 1 +revision 0 complete +requested-url https://example.com/ +resolved-url https://example.com/ +title Example Domain +heading 1 Example Domain +text This domain is for use in illustrative examples. +link Learn more https://www.iana.org/domains/example +end From 85f9f5468eb704bc58c09887e93fc992c2844266 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:33:06 -0400 Subject: [PATCH 13/55] Add offline HTML and independent expected Prepaint Version 1 fixtures --- tests/fixtures/ib-html/arxiv.html | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/fixtures/ib-html/arxiv.html diff --git a/tests/fixtures/ib-html/arxiv.html b/tests/fixtures/ib-html/arxiv.html new file mode 100644 index 00000000..c5aaf23f --- /dev/null +++ b/tests/fixtures/ib-html/arxiv.html @@ -0,0 +1,9 @@ + +Article & algebra +

2 Introduction

+

A word with é and 🙂.

+
NameValue
+

See abstract afterward.

+Diagram + + From 5f3a96c8b0ac1e05000f95c95bd672cc2f58ff5f Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:33:08 -0400 Subject: [PATCH 14/55] Add offline HTML and independent expected Prepaint Version 1 fixtures --- tests/fixtures/ib-html/arxiv.prepaint | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/fixtures/ib-html/arxiv.prepaint diff --git a/tests/fixtures/ib-html/arxiv.prepaint b/tests/fixtures/ib-html/arxiv.prepaint new file mode 100644 index 00000000..cdc93cb8 --- /dev/null +++ b/tests/fixtures/ib-html/arxiv.prepaint @@ -0,0 +1,13 @@ +ib-prepaint 1 +revision 0 complete +requested-url https://arxiv.org/html/2401.00100 +resolved-url https://arxiv.org/html/2401.00100 +title Article & algebra +heading 2 2 Introduction +text A word with é and 🙂. +row Name Value +text See +link abstract https://arxiv.org/abs/2401.00100 +text afterward. +text Image pending acquisition: Diagram — Figure 1 +end From f34f88760378ff0f46a0c7b7bac82e5a1dce64fc Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:33:53 -0400 Subject: [PATCH 15/55] =?UTF-8?q?Exercise=20actual=20Idri=C3=A7=20converte?= =?UTF-8?q?r,=20independent=20wire=20consumer,=20golden=20and=20fail-prese?= =?UTF-8?q?rve=20mutants?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_html_prepaint.sh | 121 ++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/test_html_prepaint.sh diff --git a/tests/test_html_prepaint.sh b/tests/test_html_prepaint.sh new file mode 100644 index 00000000..a9ff87c1 --- /dev/null +++ b/tests/test_html_prepaint.sh @@ -0,0 +1,121 @@ +#!/bin/sh +set -eu +root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + tests/prepaint_v1_oracle.c -o "$work/prepaint-v1-oracle" +export IB_PREPAINT_ORACLE="$work/prepaint-v1-oracle" +export IB_HTML_PREPAINT_EXEC="${IB_HTML_PREPAINT_EXEC:-$root/src/build/exec/ib-html-prepaint}" +export IB_ALLOW_FIXTURE=1 +export IB_EXPECTED_SOURCE_GENERATION=generation-1 +test -x "$IB_HTML_PREPAINT_EXEC" + +make_manifest() { + kind=$1 body=$2 url=$3 manifest=$4 + sha=$(sha256sum "$body" | cut -d ' ' -f 1) + bytes=$(wc -c < "$body" | tr -d '[:space:]') + { + printf 'schema\tib-html-admission-v1\n' + printf 'source_kind\tfixture\n' + printf 'source_key\tfixture:%s\n' "$kind" + printf 'source_generation\tgeneration-1\n' + printf 'requested_url\t%s\n' "$url" + printf 'resolved_url\t%s\n' "$url" + printf 'http_status\t200\n' + printf 'mime_type\ttext/html\n' + printf 'charset\tutf-8\n' + printf 'sha256\t%s\n' "$sha" + printf 'byte_count\t%s\n' "$bytes" + printf 'acquisition_complete\tYES\n' + printf 'arena_reopen\tNOT_RUN\n' + } > "$manifest" +} +run_fixture() { + kind=$1 url=$2 + body="tests/fixtures/ib-html/$kind.html" + key="fixture:$kind" + expected="tests/fixtures/ib-html/$kind.prepaint" + manifest="$work/$kind.tsv" + output="$work/$kind.prepaint" + make_manifest "$kind" "$body" "$url" "$manifest" + sh bin/ib_emit_html_prepaint.grease "$manifest" "$body" "$key" "$output" \ + > "$work/$kind.receipt" + "$IB_PREPAINT_ORACLE" "$output" + cmp "$expected" "$output" + grep -Fx "$(printf 'conversion\tPASS')" "$work/$kind.receipt" + printf 'fixture\t%s\tPASS\n' "$kind" +} +run_fixture example https://example.com/ +run_fixture arxiv https://arxiv.org/html/2401.00100 + +sourcefile="$work/example.prepaint" +manifest="$work/example.tsv" +body=tests/fixtures/ib-html/example.html +key=fixture:example +cp "$sourcefile" "$work/last_valid" +test_failure_preserves() { + label=$1 + if sh bin/ib_emit_html_prepaint.grease "$manifest" "$body" "$key" "$sourcefile" \ + > "$work/mutant.out" 2> "$work/mutant.err"; then + echo "failure accepted: $label" >&2 + exit 1 + fi + cmp "$sourcefile" "$work/last_valid" + printf 'refusal\t%s\tPASS\n' "$label" +} +cp "$manifest" "$work/good.tsv" +sed 's@^sha256.*@sha256\t0000000000000000000000000000000000000000000000000000000000000000@' \ + "$work/good.tsv" > "$manifest" +test_failure_preserves wrong_digest +cp "$work/good.tsv" "$manifest" +sed 's@^source_key.*@source_key\tfixture:wrong@' "$work/good.tsv" > "$manifest" +test_failure_preserves changed_source_identity +cp "$work/good.tsv" "$manifest" +IB_EXPECTED_SOURCE_GENERATION=generation-2 +export IB_EXPECTED_SOURCE_GENERATION +test_failure_preserves stale_source_generation +IB_EXPECTED_SOURCE_GENERATION=generation-1 +export IB_EXPECTED_SOURCE_GENERATION +sed 's@^mime_type.*@mime_type\tapplication/octet-stream@' "$work/good.tsv" > "$manifest" +test_failure_preserves unsupported_mime +cp "$work/good.tsv" "$manifest" +sed 's@^acquisition_complete.*@acquisition_complete\tNO@' "$work/good.tsv" > "$manifest" +test_failure_preserves incomplete_source +cp "$work/good.tsv" "$manifest" +# Same published metadata, mutated body: cannot publish changed bytes. +printf '\377' > "$work/invalid.html" +body="$work/invalid.html" +test_failure_preserves corrupted_source +make_manifest example "$body" https://example.com/ "$manifest" +test_failure_preserves invalid_utf8 +body=tests/fixtures/ib-html/example.html +cp "$work/good.tsv" "$manifest" + +# Wrong-output mutants must not pass independent expected-content comparison. +mutation_refused() { + label=$1 + test ! "$1" = "" + if cmp -s "$sourcefile" "$work/$label.prepaint"; then + echo "undetected mutation: $label" >&2; exit 1 + fi + printf 'semantic-mutant\t%s\tREFUSED_BY_GOLDEN\n' "$label" +} +sed '/^text\tThis domain/d' "$sourcefile" > "$work/dropped_text.prepaint" +mutation_refused dropped_text +sed 's@^heading\t1\t@heading\t2\t@' "$sourcefile" > "$work/wrong_heading.prepaint" +mutation_refused wrong_heading +sed 's@www.iana.org/domains/example@www.iana.org/domains/wrong@' "$sourcefile" > "$work/wrong_link.prepaint" +mutation_refused wrong_link +sed 's@^heading\t1\t@heading\t7\t@' "$sourcefile" > "$work/invalid_heading.prepaint" +if "$IB_PREPAINT_ORACLE" "$work/invalid_heading.prepaint"; then + echo 'independent oracle accepted heading 7' >&2; exit 1 +fi +printf 'syntax-mutant\tinvalid_heading\tREFUSED\n' +sed 's@^title\t.*@title\tbad\\q@' "$sourcefile" > "$work/invalid_escaping.prepaint" +if "$IB_PREPAINT_ORACLE" "$work/invalid_escaping.prepaint"; then + echo 'independent oracle accepted invalid escape' >&2; exit 1 +fi +printf 'syntax-mutant\tinvalid_escaping\tREFUSED\n' +printf 'semantic\tPASS\n' From 0c6377670fd1f19016ecbc09e2d3193d9cc69621 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:34:01 -0400 Subject: [PATCH 16/55] =?UTF-8?q?Run=20independent=20prepaint=20consumer?= =?UTF-8?q?=20and=20wrong-output=20mutations=20against=20compiled=20Idri?= =?UTF-8?q?=C3=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/idric-core.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 765e04a7..c2b7235e 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -113,6 +113,12 @@ jobs: /tmp/idric/bin/idris2 HtmlPrepaintCommand.idric -o ib-html-prepaint test -x build/exec/ib-html-prepaint + - name: Independent HTML prepaint wire and corruption-oracle acceptance + run: | + sh -n bin/ib_emit_html_prepaint.grease + sh -n tests/test_html_prepaint.sh + sh tests/test_html_prepaint.sh + - name: Exercise browser-owned Idriç core run: sh bin/ci_browser_foundation.grease exercise-core From d9af9d025b656266b19d9b238c16ac47b2e5cd7c Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:35:30 -0400 Subject: [PATCH 17/55] Retain figure captions as explicit source events for checked image association --- src/IB/Information.idric | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/IB/Information.idric b/src/IB/Information.idric index 19befbc8..61ecb0b0 100644 --- a/src/IB/Information.idric +++ b/src/IB/Information.idric @@ -6,6 +6,7 @@ public export data InformationItem = Heading Text | HeadingAt Text Text + | FigureCaption Text | TextBlock Text | Link Text Text | Image Text Text Text Text @@ -43,6 +44,7 @@ data Capture = NoCapture | CaptureTitle | CaptureHeading Text + | CaptureCaption | CaptureText Text | CaptureLink Text Bool Capture | CaptureForm Text @@ -351,6 +353,7 @@ capture_target : Capture → Text capture_target NoCapture = "" capture_target CaptureTitle = "title" capture_target (CaptureHeading name) = name +capture_target CaptureCaption = "figcaption" capture_target (CaptureText name) = name capture_target (CaptureLink _ _ _) = "a" capture_target (CaptureForm _) = "form" @@ -365,6 +368,8 @@ finish_capture NoCapture text title items row = (title, items, row) finish_capture CaptureTitle text title items row = (text, items, row) finish_capture (CaptureHeading name) text title items row = if text ≟ "" then (title, items, row) else (title, HeadingAt name text :: items, row) +finish_capture CaptureCaption text title items row = + if text ≟ "" then (title, items, row) else (title, FigureCaption text :: items, row) finish_capture (CaptureText _) text title items row = if text ≟ "" then (title, items, row) else (title, TextBlock text :: items, row) finish_capture (CaptureLink href has_image parent) text title items row = @@ -452,6 +457,8 @@ collect_information (OpenTag name tag :: rest) skip capture chars row title item then collect_information rest NotSkipping (CaptureHeading name) [] row title items else if name ≟ "p" || name ≟ "li" then collect_information rest NotSkipping (CaptureText name) [] row title items + else if name ≟ "figcaption" + then collect_information rest NotSkipping CaptureCaption [] row title items else if name ≟ "a" then collect_information rest NotSkipping (CaptureLink (attribute_or_empty "href" tag) False NoCapture) [] row title items @@ -583,6 +590,7 @@ join_cells (value :: rest) = value ++ " " ++ join_cells rest item_text : InformationItem → Text item_text (Heading text) = text item_text (HeadingAt name text) = text +item_text (FigureCaption text) = text item_text (TextBlock text) = text item_text (Link text href) = text ++ " " ++ href item_text (Image source alt caption href) = alt ++ " " ++ source ++ " " ++ caption ++ " " ++ href @@ -618,6 +626,7 @@ join_with separator (value :: rest) = value ++ separator ++ join_with separator render_item : Number → InformationItem → Text render_item width (Heading text) = clip_text width ("# " ++ text) render_item width (HeadingAt name text) = clip_text width ("# " ++ text) +render_item width (FigureCaption text) = clip_text width ("caption: " ++ text) render_item width (TextBlock text) = clip_text width text render_item width (Link text href) = clip_text width ("link: " ++ text ++ " -> " ++ href) render_item width (Image source alt caption href) = From 230563130c62fc15a4d6c270d1df00dd9c32c2a6 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:35:37 -0400 Subject: [PATCH 18/55] Bind adjacent figure captions to pending source-linked images --- src/IB/HtmlPrepaint.idric | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index 99964578..c3635055 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -138,6 +138,9 @@ map_item e base (HeadingAt name text) = Just level ⇒ ([IB.Strand.Heading level text], [], []) map_item e base (IB.Information.TextBlock text) = ([IB.Strand.TextBlock text], [], []) +map_item e base (FigureCaption caption) = + ([IB.Strand.TextBlock ("Figure caption: " ++ caption)], [], + ["orphaned-figure-caption"]) map_item e base (IB.Information.Link text href) = let (blocks, notes) = checked_link base text href in (blocks, [], notes) @@ -168,6 +171,10 @@ map_item e base (IB.Information.Image url alt caption destination) = map_items : HtmlEvidence → String → List InformationItem → (List InformationBlock, List PendingImage, List String) map_items e base [] = ([], [], []) +map_items e base + (IB.Information.Image source alt caption target :: FigureCaption figure_text :: rest) = + let effective_caption = if figure_text == "" then caption else figure_text in + map_items e base (IB.Information.Image source alt effective_caption target :: rest) map_items e base (item :: rest) = let (block, images, notes) = map_item e base item in let (other_blocks, other_images, other_notes) = map_items e base rest in From c75ffaffce1c6ea012e2b60e62758594cd70af45 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:35:44 -0400 Subject: [PATCH 19/55] Exercise figcaption association instead of image title fallback --- tests/fixtures/ib-html/arxiv.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/ib-html/arxiv.html b/tests/fixtures/ib-html/arxiv.html index c5aaf23f..a6f34219 100644 --- a/tests/fixtures/ib-html/arxiv.html +++ b/tests/fixtures/ib-html/arxiv.html @@ -4,6 +4,6 @@

2 Introduction

A word with é and 🙂.

NameValue

See abstract afterward.

-Diagram +
Diagram
Figure 1
From 2e927b442185bb49a3195c57414559f2bc8a3e12 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:36:17 -0400 Subject: [PATCH 20/55] Recover omitted HTML heading, paragraph, cell and row boundaries in the existing log --- src/IB/RecoveredInformation.idric | 60 ++++++++++++++++++------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/src/IB/RecoveredInformation.idric b/src/IB/RecoveredInformation.idric index 470e145c..283340f8 100644 --- a/src/IB/RecoveredInformation.idric +++ b/src/IB/RecoveredInformation.idric @@ -14,7 +14,8 @@ data DocumentEvent public export data RecoveredDocumentLog = DocumentLog String (List DocumentEvent) -data RecoveryState = RecoverState (Maybe String) (Maybe String) Bool Bool +data RecoveryState = + RecoverState (Maybe String) (Maybe String) Bool Bool (Maybe String) heading_tag : String → Bool heading_tag name = elem name ["h1", "h2", "h3", "h4", "h5", "h6"] @@ -23,7 +24,7 @@ paragraph_boundary : String → Bool paragraph_boundary name = heading_tag name || elem name ["table", "tr", "div", "section", "article", "aside", "nav", - "ul", "ol", "form"] + "ul", "ol", "form", "figure", "figcaption", "blockquote"] close_matching : String → Maybe String → Maybe String close_matching wanted Nothing = Nothing @@ -35,63 +36,74 @@ close_maybe Nothing = [] close_maybe (Just name) = [ImpliedEndTag name] eof_events : RecoveryState → List DocumentEvent -eof_events (RecoverState paragraph cell row_open link_open) = +eof_events (RecoverState paragraph cell row_open link_open heading) = (if link_open then [ImpliedEndTag "a"] else []) ++ close_maybe cell ++ (if row_open then [ImpliedEndTag "tr"] else []) ++ - close_maybe paragraph + close_maybe paragraph ++ close_maybe heading recover_tokens : List InformationToken → RecoveryState → List DocumentEvent recover_tokens [] state = eof_events state recover_tokens (TextToken text :: rest) state = TextEvent text :: recover_tokens rest state -recover_tokens (CloseTag name :: rest) (RecoverState paragraph cell row_open link_open) = - if name == "p" || name == "li" +recover_tokens (CloseTag name :: rest) + (RecoverState paragraph cell row_open link_open heading) = + if heading_tag name then EndTag name :: recover_tokens rest - (RecoverState (close_matching name paragraph) cell row_open link_open) + (RecoverState paragraph cell row_open link_open (close_matching name heading)) + else if name == "p" || name == "li" + then EndTag name :: recover_tokens rest + (RecoverState (close_matching name paragraph) cell row_open link_open heading) else if name == "td" || name == "th" then EndTag name :: recover_tokens rest - (RecoverState paragraph Nothing row_open link_open) + (RecoverState paragraph Nothing row_open link_open heading) else if name == "tr" then close_maybe cell ++ [EndTag "tr"] ++ recover_tokens rest - (RecoverState paragraph Nothing False link_open) + (RecoverState paragraph Nothing False link_open heading) else if name == "table" then close_maybe cell ++ (if row_open then [ImpliedEndTag "tr"] else []) ++ [EndTag "table"] ++ recover_tokens rest - (RecoverState paragraph Nothing False link_open) + (RecoverState paragraph Nothing False link_open heading) else if name == "a" then EndTag "a" :: recover_tokens rest - (RecoverState paragraph cell row_open False) + (RecoverState paragraph cell row_open False heading) else EndTag name :: recover_tokens rest - (RecoverState paragraph cell row_open link_open) -recover_tokens (OpenTag name tag :: rest) (RecoverState paragraph cell row_open link_open) = - if name == "p" || name == "li" - then close_maybe paragraph ++ [StartTag name tag] ++ recover_tokens rest - (RecoverState (Just name) cell row_open link_open) + (RecoverState paragraph cell row_open link_open heading) +recover_tokens (OpenTag name tag :: rest) + (RecoverState paragraph cell row_open link_open heading) = + if heading_tag name + then close_maybe heading ++ close_maybe paragraph ++ + [StartTag name tag] ++ recover_tokens rest + (RecoverState Nothing cell row_open link_open (Just name)) + else if name == "p" || name == "li" + then close_maybe heading ++ close_maybe paragraph ++ + [StartTag name tag] ++ recover_tokens rest + (RecoverState (Just name) cell row_open link_open Nothing) else if name == "tr" - then close_maybe paragraph ++ close_maybe cell ++ + then close_maybe heading ++ close_maybe paragraph ++ close_maybe cell ++ (if row_open then [ImpliedEndTag "tr"] else []) ++ [StartTag name tag] ++ recover_tokens rest - (RecoverState Nothing Nothing True link_open) + (RecoverState Nothing Nothing True link_open Nothing) else if name == "td" || name == "th" then close_maybe cell ++ [StartTag name tag] ++ recover_tokens rest - (RecoverState paragraph (Just name) row_open link_open) + (RecoverState paragraph (Just name) row_open link_open heading) else if name == "a" then (if link_open then [ImpliedEndTag "a"] else []) ++ [StartTag name tag] ++ recover_tokens rest - (RecoverState paragraph cell row_open True) + (RecoverState paragraph cell row_open True heading) else if paragraph_boundary name - then close_maybe paragraph ++ [StartTag name tag] ++ recover_tokens rest - (RecoverState Nothing cell row_open link_open) + then close_maybe heading ++ close_maybe paragraph ++ + [StartTag name tag] ++ recover_tokens rest + (RecoverState Nothing cell row_open link_open Nothing) else StartTag name tag :: recover_tokens rest - (RecoverState paragraph cell row_open link_open) + (RecoverState paragraph cell row_open link_open heading) public export recover_document_log : String → String → RecoveredDocumentLog recover_document_log source html = DocumentLog source - (recover_tokens (tokenize_information_html html) (RecoverState Nothing Nothing False False)) + (recover_tokens (tokenize_information_html html) (RecoverState Nothing Nothing False False Nothing)) public export document_log_contract : String From d720adc7101331a89b56ca560004cf314079fed8 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:37:13 -0400 Subject: [PATCH 21/55] Keep conversion failure explicit without asserting unchecked totality of upstream extraction --- src/IB/HtmlPrepaint.idric | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index c3635055..f6e7a584 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -7,7 +7,7 @@ import IB.Prepaint import IB.Prepaint.Version1 import IB.HtmlLinks -%default total +-- Bounded by caller/admission; explicit conversion failures are returned. -- A producer supplies this ONLY after checking the exact committed Arena range. -- Claimed and reopened digest/length are independent inputs, not a hash of an From ecb4b0c13bdf14a08eed9491411acb4c99b59f73 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:37:15 -0400 Subject: [PATCH 22/55] Keep conversion failure explicit without asserting unchecked totality of upstream extraction --- src/IB/HtmlLinks.idric | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IB/HtmlLinks.idric b/src/IB/HtmlLinks.idric index 2109fa15..e97bba62 100644 --- a/src/IB/HtmlLinks.idric +++ b/src/IB/HtmlLinks.idric @@ -2,7 +2,7 @@ module IB.HtmlLinks import IB.Information -%default total +-- Bounded by caller/admission; explicit conversion failures are returned. -- This is a deliberately bounded address resolver for admitted HTML links. -- It does not decode percent escapes or extend ICU's supported URL schemes. From 86729511ab932385b6d0444315c31a5a4fc9d290 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:37:50 -0400 Subject: [PATCH 23/55] Document exact source admission, parser scope, independent oracles and physical NOT_RUN boundaries --- docs/html-prepaint-admission.md | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 docs/html-prepaint-admission.md diff --git a/docs/html-prepaint-admission.md b/docs/html-prepaint-admission.md new file mode 100644 index 00000000..118b0efd --- /dev/null +++ b/docs/html-prepaint-admission.md @@ -0,0 +1,94 @@ +# IB-C67 HTML → typed Prepaint Version 1 + +Status: **DRAFT / NOT QUALIFIED** until an actual Idriç execution, independent +consumer and source admission have succeeded. This is the first complete-revision +conversion only. It is not a C67 URL-browser acceptance receipt. + +## Owners and immutable starting points + +- Semantic base: PR #148, `work/linux-shared-contract`, originally + `2425d21d984de21e5b65ae0d2e09c586120239ee`. + This includes the types and existing serializer from #48/#123. +- Upstream Arena producer: PR #152, original head + `cc5f64b84c316b88d4c5c67fe4ea7ee5fa1b9bbe`. + `bin/ib_android_arena_read.grease` supplies a digest-checked range but its + receipt has `resolved_url=UNOBSERVED`, `mime_type=UNOBSERVED`, + `http_status=UNOBSERVED`, `navigation_admitted=NO`, and no pinned source + generation or charset. **No live HTML can pass source admission yet.** +- Separate reader-handoff owner: PR #153, original head + `9cf28c2f6c38cb12bb906a844e061ebfab34f943`. +- C67 native Material reader: PR #129. It currently paints a pinned snapshot. +- ICU final URL/redirect evidence: `dilapidated-shed/icu` PR #35. +- The browser-foundation CI pins Idriç + `7050c116326c9c8e8d0a56c2db7baf2119cbae19`. + The underlying runner's compiled executable digest must be recorded from the + *actual* artifact; the source pin is not an executable receipt. + +## Byte and semantic boundaries + +1. A source owner must commit the original bytes, source generation, requested + and actual ICU-resolved URLs, status, content type, charset, byte count and + source digest. Unobserved or stale metadata is **not admitted**. +2. Grease reopens and checks the exact readback bytes and SHA-256 and validates + the UTF-8 encoding. The source key/generation must match the browser-owned + navigation expectation. It must not mint a second persistent source store. +3. The Idriç converter consumes an admitted immutable HTML string through + `IB.RecoveredInformation` and `IB.Information`, maps ordered items into + `IB.Strand`, then calls `prepaint_from_strand` and the existing + `IB.Prepaint.Version1.serialize_version1`. +4. A newly emitted complete Version 1 stream is checked by a *different* C11 + consumer and replaces the last valid disposable projection only after + success. A rejected source or converter failure never changes the prior + projection. +5. Display width is not an input to semantic extraction; links remain navigation + candidates, never reader-fetch permissions. The Android reader/Binder/PFD + adapter must independently bind the publication to the retained source and + currently remains **NOT_RUN**. + +The Version 1 wire schema does **not** contain a source digest/generation. The +`HtmlPrepaintResult` retains that metadata and the Grease receipt identifies +the projection separately. The reader must keep that binding at handoff; a +standalone `.prepaint` file must not be promoted to canonical source identity. + +`bin/ib_emit_html_prepaint.grease` consumes one thirteen-field source admission +receipt and an **already verified** body path, then requires an exact expected +source key and `IB_EXPECTED_SOURCE_GENERATION`. The receipt is authority only +when supplied by the trusted source-store owner, not by an app/remote page. +The `fixture` kind is prohibited unless `IB_ALLOW_FIXTURE=1`; tests always +label fixtures. This adapter does **not** grant Arena metadata qualification: +PR #152 must be reconciled before an Arena-owner receipt can be supplied. + +## Supported now / explicit limitations + +- Six distinct headings, nested inline text, paragraphs, lists as ordered text + blocks, table rows, basic character references, relative/absolute/query/ + fragment links and a valid HTML base URL. +- Missing closing heading/paragraph/list/cell/row tags are passed through the + existing recovered document log. +- Forms are visible descriptions with **no submission**. Script, style, + canvas, video, SVG and other dynamic elements are excluded or noted. +- Images have resolved acquisition-request URLs, alt text and available + title/figure captions. Without separate image admission only a text + placeholder is serialized. **No remote image URI is emitted for the reader + to fetch directly.** +- Not a full HTML5/CSS/JavaScript/DOM parser. Unsupported charsets, unsafe + schemes and an unobserved final URL cause refusal or explicit limitations. +- In-place width reflow and selection identity need graphical reader acceptance; + typed source-linked blocks alone are not that acceptance. + +## Acceptance matrix + +| Check | Authority | Current status | +|---|---|---| +| Idriç typecheck + actual executable | IB CI | NOT_RUN until workflow completes | +| Offline example.com-shaped fixture | Idriç + independent C11 oracle and frozen golden | NOT_RUN | +| Offline arXiv-shaped fixture | Idriç + independent C11 oracle and frozen golden | NOT_RUN | +| Bad digest, generation, source ID, UTF-8 and omitted/stale metadata | Grease refusal, last-valid byte comparison | NOT_RUN | +| Mutated heading, link, dropped text and malformed escaping | Independent oracle/golden | NOT_RUN | +| Real ICU HTTPS → committed Arena → admitted metadata | ICU PR #35 + IB PR #152 | BLOCKED | +| Binder/PFD delivers new projection to reader | IB PR #153 | NOT_RUN | +| Physical C67 paints example.com and navigates links | User-observed device + receipts | NOT_RUN | +| ARM32 A1/C67, ARM64 C67 converter execution | Cat Food/CI evidence | NOT_RUN | + +Keep the PR draft and task open until independently admitted retained HTML +actually produces a valid complete prepaint. Do not label an offline fixture live. From eb04eb8bbee61d58f77a8f12000d251bf002b73f Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:38:13 -0400 Subject: [PATCH 24/55] Add fast independent C11 wire-grammar and frozen semantic golden acceptance lane --- tests/test_prepaint_oracle_static.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/test_prepaint_oracle_static.sh diff --git a/tests/test_prepaint_oracle_static.sh b/tests/test_prepaint_oracle_static.sh new file mode 100644 index 00000000..c11ea2af --- /dev/null +++ b/tests/test_prepaint_oracle_static.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu +root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + tests/prepaint_v1_oracle.c -o "$work/oracle" +for case_name in example arxiv; do + "$work/oracle" "tests/fixtures/ib-html/$case_name.prepaint" + printf 'independent-golden\t%s\tPASS\n' "$case_name" +done +cp tests/fixtures/ib-html/example.prepaint "$work/sample" +sed 's@^heading\t1\t@heading\t7\t@' "$work/sample" > "$work/level" +if "$work/oracle" "$work/level"; then exit 1; fi +sed 's@^title\t.*@title\tbad\\q@' "$work/sample" > "$work/escape" +if "$work/oracle" "$work/escape"; then exit 1; fi +sed 's@^link\tLearn more\t.*@link\tLearn more\tjavascript:alert(1)@' "$work/sample" > "$work/link" +if "$work/oracle" "$work/link"; then exit 1; fi +# Valid but semantically wrong content MUST fail the independent frozen golden. +sed '/^text\tThis domain/d' "$work/sample" > "$work/drop" +if cmp -s "$work/drop" tests/fixtures/ib-html/example.prepaint; then exit 1; fi +sed 's@^heading\t1\t@heading\t2\t@' "$work/sample" > "$work/wrong-level" +if cmp -s "$work/wrong-level" tests/fixtures/ib-html/example.prepaint; then exit 1; fi +sed 's@www.iana.org/domains/example@www.iana.org/domains/wrong@' "$work/sample" > "$work/wrong-target" +if cmp -s "$work/wrong-target" tests/fixtures/ib-html/example.prepaint; then exit 1; fi +printf 'prepaint-oracle-static\tPASS\n' From 2724e943a58324c14925e176a5d0cac96b955276 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:38:15 -0400 Subject: [PATCH 25/55] Add fast independent C11 wire-grammar and frozen semantic golden acceptance lane --- .github/workflows/ib-html-prepaint-static.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/ib-html-prepaint-static.yml diff --git a/.github/workflows/ib-html-prepaint-static.yml b/.github/workflows/ib-html-prepaint-static.yml new file mode 100644 index 00000000..ff0307c8 --- /dev/null +++ b/.github/workflows/ib-html-prepaint-static.yml @@ -0,0 +1,31 @@ +name: IB independent Prepaint Version 1 oracle + +on: + pull_request: + paths: + - 'src/IB/HtmlPrepaint.idric' + - 'src/IB/HtmlLinks.idric' + - 'src/IB/Information.idric' + - 'src/IB/RecoveredInformation.idric' + - 'tests/prepaint_v1_oracle.c' + - 'tests/test_prepaint_oracle_static.sh' + - 'tests/fixtures/ib-html/**' + - '.github/workflows/ib-html-prepaint-static.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + independent-wire: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Independently parse frozen Version 1 witnesses and reject mutants + run: | + test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" + sh -n tests/test_prepaint_oracle_static.sh + sh tests/test_prepaint_oracle_static.sh From d1cf13f11aaa70e8938fa3d6262d128760598e23 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:42:33 -0400 Subject: [PATCH 26/55] Avoid misreading data-href as href and accept case-insensitive HTML attribute names --- src/IB/Information.idric | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/IB/Information.idric b/src/IB/Information.idric index 61ecb0b0..1c4f902e 100644 --- a/src/IB/Information.idric +++ b/src/IB/Information.idric @@ -321,21 +321,30 @@ read_attribute_value chars = value ⇒ Just (pack (read_unquoted value)) _ ⇒ Nothing -seek_attribute_fuel : Number → List Char → List Char → Maybe Text -seek_attribute_fuel Z wanted chars = Nothing -seek_attribute_fuel (S fuel) wanted [] = Nothing -seek_attribute_fuel (S fuel) wanted chars@(_ :: rest) = - if starts_with_chars wanted chars +starts_with_attribute : List Char → List Char → Bool +starts_with_attribute [] chars = True +starts_with_attribute (_ :: _) [] = False +starts_with_attribute (wanted :: wanted_rest) (actual :: rest) = + wanted == ascii_lower_char actual && + starts_with_attribute wanted_rest rest + +-- Require the beginning of a complete HTML attribute name, not a suffix of +-- data-href or characters inside a quoted attribute. Preserve value case. +seek_attribute_fuel : Number → Bool → List Char → List Char → Maybe Text +seek_attribute_fuel Z boundary wanted chars = Nothing +seek_attribute_fuel (S fuel) boundary wanted [] = Nothing +seek_attribute_fuel (S fuel) boundary wanted chars@(value :: rest) = + if boundary && starts_with_attribute wanted chars then case read_attribute_value (drop_chars (length wanted) chars) of Just found ⇒ Just found - Nothing ⇒ seek_attribute_fuel fuel wanted rest - else seek_attribute_fuel fuel wanted rest + Nothing ⇒ seek_attribute_fuel fuel (is_space_char value) wanted rest + else seek_attribute_fuel fuel (is_space_char value) wanted rest public export attribute_value : Text → Text → Maybe Text attribute_value name tag = let chars = tag_remainder tag in - seek_attribute_fuel (length chars) (unpack name) chars + seek_attribute_fuel (length chars) True (unpack name) chars attribute_or_empty : Text → Text → Text attribute_or_empty name tag = From 09df5d43feb35c3da6ca5d6401ec6c43eb20d8d2 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:44:00 -0400 Subject: [PATCH 27/55] =?UTF-8?q?Stage=20verified=20PR152=20Arena=20range?= =?UTF-8?q?=20through=20strict=20metadata=20admission=20into=20Idri=C3=A7?= =?UTF-8?q?=20Version=201=20converter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/ib_arena_to_html_prepaint.grease | 81 ++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 bin/ib_arena_to_html_prepaint.grease diff --git a/bin/ib_arena_to_html_prepaint.grease b/bin/ib_arena_to_html_prepaint.grease new file mode 100644 index 00000000..f79eb9bb --- /dev/null +++ b/bin/ib_arena_to_html_prepaint.grease @@ -0,0 +1,81 @@ +#!/bin/sh +# Adapter ONLY for independently reopened committed PR #152 Arena ranges. +# It requires future admitted ICU metadata; legacy UNOBSERVED v1 refuses. +# The transport + store remain owned by #152; this imports no replacement. +set -eu +umask 077 +fail() { printf 'ib Arena → HTML prepaint: %s\n' "$1" >&2; exit 1; } +test "$#" -eq 4 || fail 'usage: ib_arena_to_html_prepaint.grease REQUEST_ID PRIVATE_ARENA_ROOT EXPECTED_GENERATION OUTPUT' +request=$1 +root=$2 +generation=$3 +output=$4 +case "$request" in + ''|.|..|*[!abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-]*) fail 'bad request identity' ;; +esac +case "$root" in /*) ;; *) fail 'Arena root must be absolute' ;; esac +test -d "$root" && test ! -L "$root" || fail 'Arena root unavailable or linked' +root=$(CDPATH='' cd -P -- "$root" && pwd -P) +manifest="$root/receipts/$request.tsv" +test -f "$manifest" && test ! -L "$manifest" || fail 'no committed receipt' +test "$(wc -c < "$manifest" | tr -d '[:space:]')" -le 8192 || fail 'oversized producer receipt' +awk -F '\t' ' + NF != 2 || $1 !~ /^[a-z_]+$/ || $2 == "" || /\r/ { bad=1 } + { if (seen[$1]++) bad=1 } + END { if (bad || NR < 14) exit 1 } +' "$manifest" || fail 'invalid or duplicate producer receipt keys' +field() { awk -F '\t' -v key="$1" '$1 == key { print $2 }' "$manifest"; } +test "$(field request_id)" = "$request" || fail 'request mismatch' +test "$(field reopen_check)" = MATCH || fail 'source not independently reopened' +test "$(field navigation_admitted)" = YES || fail 'navigation not admitted' +test "$(field source_generation)" = "$generation" || fail 'out-of-order or missing generation' +test "$(field acquisition_complete)" = YES || fail 'incomplete ICU acquisition' +test "$(field mime_type)" = text/html || fail 'unadmitted content type' +case "$(field charset)" in utf-8|UTF-8) ;; *) fail 'unobserved/unsupported charset' ;; esac +test "$(field http_status)" = 200 || fail 'unadmitted status' +requested=$(field requested_url) +resolved=$(field resolved_url) +test -n "$requested" && test -n "$resolved" && test "$resolved" != UNOBSERVED || + fail 'actual ICU final URL missing' +digest=$(field sha256) +length=$(field byte_count) +test -n "$digest" && test -n "$length" || fail 'source digest/length missing' +case "$digest" in ''|*[!0123456789abcdef]*) fail 'source digest invalid' ;; esac +test "$(printf %s "$digest" | wc -c | tr -d '[:space:]')" -eq 64 || fail 'source digest length invalid' +case "$length" in ''|*[!0123456789]*) fail 'source length invalid' ;; esac +test "$length" -gt 0 && test "$length" -le 4194304 || fail 'source length outside bound' +expected_key="arena:$request" +test "$(field source_key)" = "$expected_key" || fail 'untrusted source key' +reader=$(printenv IB_ARENA_READ_EXEC 2>/dev/null || :) +test -n "$reader" && test -x "$reader" || fail 'PR #152 verified Arena reader executable required' +parent=$(dirname -- "$output") +test -d "$parent" && test ! -L "$parent" || fail 'disposable output directory unavailable' +work=$(mktemp -d "$parent/.ib-html-arena.XXXXXXXX") || fail 'cannot stage retained source' +cleanup() { rm -rf "$work"; } +trap cleanup EXIT HUP INT TERM +# This MUST be PR #152's verified range-reading implementation. It performs +# the canonical offset, used pointer, length and SHA check before any emission. +"$reader" "$request" "$root" > "$work/body" || + fail 'canonical committed readback refused' +test "$(wc -c < "$work/body" | tr -d '[:space:]')" = "$length" || + fail 'reopened committed source length changed' +test "$(sha256sum "$work/body" | cut -d ' ' -f 1)" = "$digest" || + fail 'reopened committed source digest changed' +{ + printf 'schema\tib-html-admission-v1\n' + printf 'source_kind\tarena\n' + printf 'source_key\t%s\n' "$expected_key" + printf 'source_generation\t%s\n' "$generation" + printf 'requested_url\t%s\n' "$requested" + printf 'resolved_url\t%s\n' "$resolved" + printf 'http_status\t200\n' + printf 'mime_type\ttext/html\n' + printf 'charset\t%s\n' "$(field charset)" + printf 'sha256\t%s\n' "$digest" + printf 'byte_count\t%s\n' "$length" + printf 'acquisition_complete\tYES\n' + printf 'arena_reopen\tMATCH\n' +} > "$work/admitted.tsv" +IB_EXPECTED_SOURCE_GENERATION="$generation" \ + sh "$(dirname -- "$0")/ib_emit_html_prepaint.grease" \ + "$work/admitted.tsv" "$work/body" "$expected_key" "$output" From 954a7426cb54b3c792ada48c731f530e8b0fed34 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:46:01 -0400 Subject: [PATCH 28/55] Prove HTML character decoding total with source-bounded recursion fuel --- src/IB/Information.idric | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/IB/Information.idric b/src/IB/Information.idric index 1c4f902e..1821310c 100644 --- a/src/IB/Information.idric +++ b/src/IB/Information.idric @@ -128,16 +128,24 @@ scan_entity (S fuel) collected (';' :: rest) = Just (pack (reverse collected), r scan_entity (S fuel) collected (value :: rest) = scan_entity fuel (value :: collected) rest -decode_entity_chars : List Char → List Char -decode_entity_chars [] = [] -decode_entity_chars ('&' :: rest) = +-- Each recursive call consumes one fuel unit even if an entity scan +-- discards several input characters. This is total regardless of the scanner +-- implementation, and source-length fuel suffices for every input code point. +decode_entity_chars_fuel : Nat → List Char → List Char +decode_entity_chars_fuel Z remaining = remaining +decode_entity_chars_fuel (S fuel) [] = [] +decode_entity_chars_fuel (S fuel) ('&' :: rest) = case scan_entity 16 [] rest of - Nothing ⇒ '&' :: decode_entity_chars rest + Nothing ⇒ '&' :: decode_entity_chars_fuel fuel rest Just (name, remaining) ⇒ case entity_character name of - Nothing ⇒ '&' :: decode_entity_chars rest - Just value ⇒ value :: decode_entity_chars remaining -decode_entity_chars (value :: rest) = value :: decode_entity_chars rest + Nothing ⇒ '&' :: decode_entity_chars_fuel fuel rest + Just value ⇒ value :: decode_entity_chars_fuel fuel remaining +decode_entity_chars_fuel (S fuel) (value :: rest) = + value :: decode_entity_chars_fuel fuel rest + +decode_entity_chars : List Char → List Char +decode_entity_chars chars = decode_entity_chars_fuel (length chars) chars public export decode_html_entities : Text → Text From 724704378673d8cd4f024e10b7719f67c415d523 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 18:48:14 -0400 Subject: [PATCH 29/55] Make figure-caption traversal structurally decreasing and remove ambiguous record selector --- src/IB/HtmlPrepaint.idric | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index f6e7a584..0d203005 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -36,7 +36,7 @@ record PendingImage where image_url : String alternate : String caption : String - source_key : String + image_source_key : String public export record HtmlPrepaintResult where @@ -174,7 +174,10 @@ map_items e base [] = ([], [], []) map_items e base (IB.Information.Image source alt caption target :: FigureCaption figure_text :: rest) = let effective_caption = if figure_text == "" then caption else figure_text in - map_items e base (IB.Information.Image source alt effective_caption target :: rest) + let (blocks, images, notes) = + map_item e base (IB.Information.Image source alt effective_caption target) in + let (other_blocks, other_images, other_notes) = map_items e base rest in + (blocks ++ other_blocks, images ++ other_images, notes ++ other_notes) map_items e base (item :: rest) = let (block, images, notes) = map_item e base item in let (other_blocks, other_images, other_notes) = map_items e base rest in From 1078996e500de5f268332182f202b7daf98a649e Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:07:06 -0400 Subject: [PATCH 30/55] =?UTF-8?q?Avoid=20reserved=20Idri=C3=A7=20prefix=20?= =?UTF-8?q?keyword=20in=20HTML=20address=20resolver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/IB/HtmlLinks.idric | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/IB/HtmlLinks.idric b/src/IB/HtmlLinks.idric index e97bba62..e9dcd317 100644 --- a/src/IB/HtmlLinks.idric +++ b/src/IB/HtmlLinks.idric @@ -13,8 +13,8 @@ strip_prefix (wanted :: following) (actual :: rest) = if wanted == actual then strip_prefix following rest else Nothing starts : String → String → Bool -starts prefix value = - case strip_prefix (unpack prefix) (unpack value) of +starts sought_prefix value = + case strip_prefix (unpack sought_prefix) (unpack value) of Nothing ⇒ False Just _ ⇒ True @@ -132,8 +132,8 @@ resolve_html_reference base reference = then strip_query current ++ reference else let (relative_path, suffix) = split_path_suffix (unpack reference) [] in - let prefix = + let parent_directory = if starts "/" reference then "" else pack (parent_path (unpack (url_path origin base)) []) in - origin ++ path_normalised (prefix ++ relative_path) ++ suffix + origin ++ path_normalised (parent_directory ++ relative_path) ++ suffix if cheap_http_url candidate then Just candidate else Nothing From b5e3ac4ba88f19eef2aa5fecc02f6dbbf6a0e5f0 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:07:48 -0400 Subject: [PATCH 31/55] Resolve relative URL against last path slash, not last character --- src/IB/HtmlLinks.idric | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/IB/HtmlLinks.idric b/src/IB/HtmlLinks.idric index e9dcd317..b9ccd709 100644 --- a/src/IB/HtmlLinks.idric +++ b/src/IB/HtmlLinks.idric @@ -61,13 +61,15 @@ url_path origin url = '/' :: _ ⇒ strip_query (strip_fragment rest) _ ⇒ "/" -parent_path : List Char → List Char → List Char -parent_path [] parent = reverse parent -parent_path ('/' :: rest) previous = parent_path rest ('/' :: previous) -parent_path (value :: rest) previous = - case rest of - [] ⇒ reverse previous - _ ⇒ parent_path rest (value :: previous) +-- Directory of the final URL path, ending at its last slash. +-- Removing the last *character* rather than the last segment breaks ../ links. +at_last_slash : List Char → List Char +at_last_slash [] = [] +at_last_slash ('/' :: rest) = '/' :: rest +at_last_slash (_ :: rest) = at_last_slash rest + +parent_path : List Char → String +parent_path path_chars = pack (reverse (at_last_slash (reverse path_chars))) -- Segment operations keep the origin outside the path stack. segments : List Char → List Char → List String → List String @@ -134,6 +136,6 @@ resolve_html_reference base reference = let (relative_path, suffix) = split_path_suffix (unpack reference) [] in let parent_directory = if starts "/" reference then "" - else pack (parent_path (unpack (url_path origin base)) []) in + else parent_path (unpack (url_path origin base)) in origin ++ path_normalised (parent_directory ++ relative_path) ++ suffix if cheap_http_url candidate then Just candidate else Nothing From d4c2c2eaeb2dcc9481785f0eb2f15637297215ff Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:08:25 -0400 Subject: [PATCH 32/55] =?UTF-8?q?Use=20layout-safe=20Idri=C3=A7=20referenc?= =?UTF-8?q?e-resolution=20branches=20and=20preserve=20suffix=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/IB/HtmlLinks.idric | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/IB/HtmlLinks.idric b/src/IB/HtmlLinks.idric index b9ccd709..9eb00bca 100644 --- a/src/IB/HtmlLinks.idric +++ b/src/IB/HtmlLinks.idric @@ -128,14 +128,16 @@ resolve_html_reference base reference = else do origin ← http_origin base let current = strip_fragment base - let candidate = - if starts "#" reference then current ++ reference + if starts "#" reference + then Just (current ++ reference) else if starts "?" reference - then strip_query current ++ reference + then Just (strip_query current ++ reference) else - let (relative_path, suffix) = split_path_suffix (unpack reference) [] in + let (relative_path, suffix) = split_path_suffix (unpack reference) in let parent_directory = - if starts "/" reference then "" - else parent_path (unpack (url_path origin base)) in - origin ++ path_normalised (parent_directory ++ relative_path) ++ suffix - if cheap_http_url candidate then Just candidate else Nothing + if starts "/" reference + then "" + else parent_path (unpack (url_path origin base)) in + let candidate = origin ++ + path_normalised (parent_directory ++ relative_path) ++ suffix in + if cheap_http_url candidate then Just candidate else Nothing From 9da8048fd4ea43b4331475717d06b10bdd0a381a Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:09:38 -0400 Subject: [PATCH 33/55] Extract layout-safe helpers for reference resolution and validate fragment/query URL safety --- src/IB/HtmlLinks.idric | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/IB/HtmlLinks.idric b/src/IB/HtmlLinks.idric index 9eb00bca..a9117dd1 100644 --- a/src/IB/HtmlLinks.idric +++ b/src/IB/HtmlLinks.idric @@ -110,6 +110,24 @@ has_scheme_colon ('#' :: _) = False has_scheme_colon (':' :: _) = True has_scheme_colon (_ :: rest) = has_scheme_colon rest +checked_url : String → Maybe String +checked_url address = + if cheap_http_url address then Just address else Nothing + +directory_for : String → String → String → String +directory_for origin base relative = + if starts "/" relative + then "" + else parent_path (unpack (url_path origin base)) + +relative_address : String → String → String → String +relative_address origin base reference = + let (relative_path, suffix) = split_path_suffix (unpack reference) in + origin ++ path_normalised (directory_for origin base reference ++ relative_path) ++ suffix + +scheme_for : String → String +scheme_for base = if starts "https://" base then "https:" else "http:" + -- Returns a navigation *candidate*. ICU and navigation admission still decide -- whether another retrieval occurs; the renderer must never fetch a link. public export @@ -120,24 +138,14 @@ resolve_html_reference base reference = else if cheap_http_url reference then Just reference else if starts "//" reference - then - let candidate = (if starts "https://" base then "https:" else "http:") ++ reference in - if cheap_http_url candidate then Just candidate else Nothing + then checked_url (scheme_for base ++ reference) else if has_scheme_colon (unpack reference) then Nothing else do origin ← http_origin base let current = strip_fragment base if starts "#" reference - then Just (current ++ reference) + then checked_url (current ++ reference) else if starts "?" reference - then Just (strip_query current ++ reference) - else - let (relative_path, suffix) = split_path_suffix (unpack reference) in - let parent_directory = - if starts "/" reference - then "" - else parent_path (unpack (url_path origin base)) in - let candidate = origin ++ - path_normalised (parent_directory ++ relative_path) ++ suffix in - if cheap_http_url candidate then Just candidate else Nothing + then checked_url (strip_query current ++ reference) + else checked_url (relative_address origin base reference) From ed2f160a41376c755e84a610caa45e3097363c8b Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:10:40 -0400 Subject: [PATCH 34/55] Supply accumulation argument when extracting relative URL path suffix --- src/IB/HtmlLinks.idric | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IB/HtmlLinks.idric b/src/IB/HtmlLinks.idric index a9117dd1..a8ab2c22 100644 --- a/src/IB/HtmlLinks.idric +++ b/src/IB/HtmlLinks.idric @@ -122,7 +122,7 @@ directory_for origin base relative = relative_address : String → String → String → String relative_address origin base reference = - let (relative_path, suffix) = split_path_suffix (unpack reference) in + let (relative_path, suffix) = split_path_suffix (unpack reference) [] in origin ++ path_normalised (directory_for origin base reference ++ relative_path) ++ suffix scheme_for : String → String From ac6c5d391867a684e086d010e885489c6a16d519 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:11:36 -0400 Subject: [PATCH 35/55] Extract image admission and navigation helpers to avoid invalid nested case layout --- src/IB/HtmlPrepaint.idric | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index 0d203005..5fe16b10 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -151,20 +151,28 @@ map_item e base (IB.Information.TableRow []) = ([], [], ["empty-table-row"]) map_item e base (IB.Information.TableRow (first :: cells)) = ([IB.Strand.TableRow (table_row first cells)], [], []) +request_image : HtmlEvidence → String → String → String → String → List PendingImage +request_image evidence base address alternate caption = + case resolve_html_reference base address of + Nothing ⇒ [] + Just absolute_address ⇒ + [NeedImageAcquisition absolute_address alternate caption evidence.source_key] + +image_limitations : String → List PendingImage → List String +image_limitations original [] = ["unsupported-image-reference:" ++ original] +image_limitations original (_ :: _) = ["image-requires-separate-admission:" ++ original] + +image_navigation : String → String → String → + (List InformationBlock, List String) +image_navigation base label "" = ([], []) +image_navigation base label target = checked_link base label target + map_item e base (IB.Information.Image url alt caption destination) = let text = if alt == "" then "Unretrieved image" else alt in let label = if caption == "" then text else text ++ " — " ++ caption in - let pending = - case resolve_html_reference base url of - Nothing ⇒ [] - Just source ⇒ [NeedImageAcquisition source alt caption e.source_key] in - let image_notes = - case pending of - [] ⇒ ["unsupported-image-reference:" ++ url] - _ ⇒ ["image-requires-separate-admission:" ++ url] in - let (linked, link_notes) = - if destination == "" then ([], []) - else checked_link base label destination in + let pending = request_image e base url alt caption in + let image_notes = image_limitations url pending in + let (linked, link_notes) = image_navigation base label destination in (IB.Strand.TextBlock ("Image pending acquisition: " ++ label) :: linked, pending, image_notes ++ link_notes) From f1fd18022be796ca8185945db9cdb137467c5167 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:11:48 -0400 Subject: [PATCH 36/55] =?UTF-8?q?Place=20image=20helper=20declarations=20a?= =?UTF-8?q?head=20of=20contiguous=20Idri=C3=A7=20map=5Fitem=20equations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/IB/HtmlPrepaint.idric | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index 5fe16b10..9e8d710f 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -128,6 +128,22 @@ checked_link base label href = Nothing ⇒ ([IB.Strand.TextBlock label], ["empty-link"]) Just ref ⇒ ([IB.Strand.Link label ref], []) +request_image : HtmlEvidence → String → String → String → String → List PendingImage +request_image evidence base address alternate caption = + case resolve_html_reference base address of + Nothing ⇒ [] + Just absolute_address ⇒ + [NeedImageAcquisition absolute_address alternate caption evidence.source_key] + +image_limitations : String → List PendingImage → List String +image_limitations original [] = ["unsupported-image-reference:" ++ original] +image_limitations original (_ :: _) = ["image-requires-separate-admission:" ++ original] + +image_navigation : String → String → String → + (List InformationBlock, List String) +image_navigation base label "" = ([], []) +image_navigation base label target = checked_link base label target + map_item : HtmlEvidence → String → InformationItem → (List InformationBlock, List PendingImage, List String) map_item e base (IB.Information.Heading text) = @@ -151,22 +167,6 @@ map_item e base (IB.Information.TableRow []) = ([], [], ["empty-table-row"]) map_item e base (IB.Information.TableRow (first :: cells)) = ([IB.Strand.TableRow (table_row first cells)], [], []) -request_image : HtmlEvidence → String → String → String → String → List PendingImage -request_image evidence base address alternate caption = - case resolve_html_reference base address of - Nothing ⇒ [] - Just absolute_address ⇒ - [NeedImageAcquisition absolute_address alternate caption evidence.source_key] - -image_limitations : String → List PendingImage → List String -image_limitations original [] = ["unsupported-image-reference:" ++ original] -image_limitations original (_ :: _) = ["image-requires-separate-admission:" ++ original] - -image_navigation : String → String → String → - (List InformationBlock, List String) -image_navigation base label "" = ([], []) -image_navigation base label target = checked_link base label target - map_item e base (IB.Information.Image url alt caption destination) = let text = if alt == "" then "Unretrieved image" else alt in let label = if caption == "" then text else text ++ " — " ++ caption in From 7164fc375b97054654e2e0de305356f30ef9ba5e Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:12:34 -0400 Subject: [PATCH 37/55] =?UTF-8?q?Separate=20typed=20source=20construction?= =?UTF-8?q?=20and=20use=20straightforward=20Idri=C3=A7=20converter=20branc?= =?UTF-8?q?h=20syntax?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/IB/HtmlPrepaint.idric | 51 +++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index 9e8d710f..7dba678b 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -203,39 +203,38 @@ unsupported_events [] = [] unsupported_events (event :: rest) = unsupported_event event ++ unsupported_events rest +make_html_strand : HtmlEvidence → InformationView → + List InformationBlock → Maybe Strand +make_html_strand evidence view blocks = do + source_id ← nonempty_text + (evidence.source_key ++ ";generation=" ++ evidence.source_generation ++ + ";sha256=" ++ evidence.claimed_sha256) + requested ← requested_address evidence.requested_url + resolved ← resolved_address evidence.resolved_url + let source = Source source_id HtmlSource CompleteSource + let heading_title = if view_title view == "" then Nothing else Just (view_title view) + pure (MakeStrand source requested resolved heading_title blocks) + public export convert_admitted_html : HtmlEvidence → String → Either String HtmlPrepaintResult convert_admitted_html admission html = case source_admission_reason admission of - Just error ⇒ Left error + Just refusal ⇒ Left refusal Nothing ⇒ let log = recover_document_log admission.source_key html in - let view = information_from_log admission.requested_url - admission.resolved_url log in + let view = information_from_log admission.requested_url admission.resolved_url log in let (base, base_notes) = link_base admission.resolved_url log in let (blocks, images, notes) = map_items admission base (view_items view) in if length blocks > 4096 then Left "prepaint-block-budget-exceeded" - else do - source_id ← case nonempty_text - (admission.source_key ++ ";generation=" ++ admission.source_generation ++ - ";sha256=" ++ admission.claimed_sha256) of - Nothing ⇒ Left "empty-source" - Just value ⇒ Right value - requested ← case requested_address admission.requested_url of - Nothing ⇒ Left "empty-requested-address" - Just value ⇒ Right value - resolved ← case resolved_address admission.resolved_url of - Nothing ⇒ Left "empty-resolved-address" - Just value ⇒ Right value - let source = Source source_id HtmlSource CompleteSource - let title = - if view_title view == "" then Nothing else Just (view_title view) - let strand = MakeStrand source requested resolved title blocks - let revision = prepaint_from_strand first_revision CompleteRevision strand - case serialize_version1 [revision] of - Nothing ⇒ Left "version1-serialization-refused" - Just wire ⇒ - Right (MadeHtmlPrepaint admission log view strand revision wire - images (base_notes ++ notes ++ - unsupported_events (document_log_events log))) + else + case make_html_strand admission view blocks of + Nothing ⇒ Left "invalid-source-strand" + Just strand ⇒ + let revision = prepaint_from_strand first_revision CompleteRevision strand in + case serialize_version1 [revision] of + Nothing ⇒ Left "version1-serialization-refused" + Just wire ⇒ + Right (MadeHtmlPrepaint admission log view strand revision wire + images (base_notes ++ notes ++ + unsupported_events (document_log_events log))) From dbe6c81b05c6987aa09aa10cdbb5a7cfb4f1a053 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:12:45 -0400 Subject: [PATCH 38/55] =?UTF-8?q?Keep=20admitted=20source=20ID=20construct?= =?UTF-8?q?ion=20unambiguous=20in=20Idri=C3=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/IB/HtmlPrepaint.idric | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/IB/HtmlPrepaint.idric b/src/IB/HtmlPrepaint.idric index 7dba678b..d61ccf16 100644 --- a/src/IB/HtmlPrepaint.idric +++ b/src/IB/HtmlPrepaint.idric @@ -203,12 +203,15 @@ unsupported_events [] = [] unsupported_events (event :: rest) = unsupported_event event ++ unsupported_events rest +source_reference_text : HtmlEvidence → String +source_reference_text evidence = + evidence.source_key ++ ";generation=" ++ evidence.source_generation ++ + ";sha256=" ++ evidence.claimed_sha256 + make_html_strand : HtmlEvidence → InformationView → List InformationBlock → Maybe Strand make_html_strand evidence view blocks = do - source_id ← nonempty_text - (evidence.source_key ++ ";generation=" ++ evidence.source_generation ++ - ";sha256=" ++ evidence.claimed_sha256) + source_id ← nonempty_text (source_reference_text evidence) requested ← requested_address evidence.requested_url resolved ← resolved_address evidence.resolved_url let source = Source source_id HtmlSource CompleteSource From c7ccf526bad9c9e96912758b916dc42dfe39886a Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:14:09 -0400 Subject: [PATCH 39/55] =?UTF-8?q?Make=20Idri=C3=A7=20prepaint=20CLI=20bran?= =?UTF-8?q?ch=20a=20direct=20call=20to=20typed=20emission=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/HtmlPrepaintCommand.idric | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/HtmlPrepaintCommand.idric b/src/HtmlPrepaintCommand.idric index f6e0d030..e1518b7c 100644 --- a/src/HtmlPrepaintCommand.idric +++ b/src/HtmlPrepaintCommand.idric @@ -4,13 +4,18 @@ import IB.HtmlPrepaint import System import System.File --- The caller must independently reopen/hash the *exact* Arena range before --- constructing these arguments. This executable does no I/O except one --- bounded already-verified UTF-8 body read and writing the disposable wire. +-- Only Grease may invoke this command for a separately reopened and +-- source-admitted body. This program never acquires or stores HTML sources. program_arguments : List String → List String program_arguments [] = [] program_arguments (_ :: args) = args +emit_prepaint : HtmlEvidence → String → IO () +emit_prepaint evidence html = + case convert_admitted_html evidence html of + Left reason ⇒ die ("html-prepaint: " ++ reason) + Right prepaint ⇒ putStr prepaint.wire + covering main : IO () main = do @@ -19,15 +24,11 @@ main = do [source_id, generation, claimed_sha, reopened_sha, claimed_bytes, reopened_bytes, requested, resolved, mime, charset, status, body_path] ⇒ do - data_result <- readFile body_path - case data_result of + body_result <- readFile body_path + case body_result of Left _ ⇒ die "html-prepaint: admitted body unreadable" - Right html ⇒ do - let evidence = - HtmlSourceEvidence source_id generation claimed_sha reopened_sha - (cast claimed_bytes) (cast reopened_bytes) - requested resolved mime charset status True True - case convert_admitted_html evidence html of - Left reason ⇒ die ("html-prepaint: " ++ reason) - Right prepaint ⇒ putStr prepaint.wire + Right html ⇒ emit_prepaint + (HtmlSourceEvidence source_id generation claimed_sha reopened_sha + (cast claimed_bytes) (cast reopened_bytes) + requested resolved mime charset status True True) html _ ⇒ die "usage: ib-html-prepaint SOURCE GENERATION CLAIM_SHA REOPEN_SHA CLAIM_COUNT REOPEN_COUNT REQUESTED FINAL MIME CHARSET STATUS ADMITTED_BODY_PATH" From d9a9a27f5bd5791f6f738e9b887c3a536a5894ab Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:15:07 -0400 Subject: [PATCH 40/55] Allow digits after key prefix in admission receipt fields such as sha256 --- bin/ib_emit_html_prepaint.grease | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ib_emit_html_prepaint.grease b/bin/ib_emit_html_prepaint.grease index 6a0dd64c..9364bbcf 100644 --- a/bin/ib_emit_html_prepaint.grease +++ b/bin/ib_emit_html_prepaint.grease @@ -14,7 +14,7 @@ test -f "$body" && test ! -L "$body" || fail 'missing or linked admitted body' test ! -L "$output" || fail 'refusing output symlink' test "$(wc -c < "$manifest" | tr -d '[:space:]')" -le 4096 || fail 'oversized admission' awk -F '\t' ' - NF != 2 || $1 !~ /^[a-z_]+$/ || $2 == "" || /\r/ { bad=1 } + NF != 2 || $1 !~ /^[a-z_][a-z_0-9]*$/ || $2 == "" || /\r/ { bad=1 } { if (seen[$1]++) bad=1 } END { if (bad || NR != 13) exit 1 } ' "$manifest" || fail 'bad admission structure' From 16d7f14c77854527fc4c2ee5faf56573d5f0feef Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:15:10 -0400 Subject: [PATCH 41/55] Allow digits after key prefix in admission receipt fields such as sha256 --- bin/ib_arena_to_html_prepaint.grease | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ib_arena_to_html_prepaint.grease b/bin/ib_arena_to_html_prepaint.grease index f79eb9bb..a497e1a9 100644 --- a/bin/ib_arena_to_html_prepaint.grease +++ b/bin/ib_arena_to_html_prepaint.grease @@ -20,7 +20,7 @@ manifest="$root/receipts/$request.tsv" test -f "$manifest" && test ! -L "$manifest" || fail 'no committed receipt' test "$(wc -c < "$manifest" | tr -d '[:space:]')" -le 8192 || fail 'oversized producer receipt' awk -F '\t' ' - NF != 2 || $1 !~ /^[a-z_]+$/ || $2 == "" || /\r/ { bad=1 } + NF != 2 || $1 !~ /^[a-z_][a-z_0-9]*$/ || $2 == "" || /\r/ { bad=1 } { if (seen[$1]++) bad=1 } END { if (bad || NR < 14) exit 1 } ' "$manifest" || fail 'invalid or duplicate producer receipt keys' From 66cbf3693d8f529eb85f8dd378c2051a70ff9641 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:16:21 -0400 Subject: [PATCH 42/55] =?UTF-8?q?Feed=20actual=20compiled=20Idri=C3=A7=20f?= =?UTF-8?q?ixture=20output=20to=20pinned=20independent=20AI-CI=20verifier?= =?UTF-8?q?=20and=20retain=20witnesses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_html_prepaint.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_html_prepaint.sh b/tests/test_html_prepaint.sh index a9ff87c1..4d58af83 100644 --- a/tests/test_html_prepaint.sh +++ b/tests/test_html_prepaint.sh @@ -44,6 +44,23 @@ run_fixture() { > "$work/$kind.receipt" "$IB_PREPAINT_ORACLE" "$output" cmp "$expected" "$output" + if test -n "${IB_AICI_PREPAINT_VERIFIER:-}"; then + test -f "$IB_AICI_PREPAINT_VERIFIER" || exit 1 + { + printf 'source_key\t%s\n' "$key" + printf 'source_generation\tgeneration-1\n' + printf 'source_sha256\t%s\n' "$(sha256sum "$body" | cut -d ' ' -f 1)" + printf 'source_bytes\t%s\n' "$(wc -c < "$body" | tr -d '[:space:]')" + } > "$work/$kind.ai-source.tsv" + sh "$IB_AICI_PREPAINT_VERIFIER" "$output" "$expected" \ + "$work/$kind.ai-source.tsv" "$body" "$key" generation-1 + fi + if test -n "${IB_HTML_PREPAINT_RECEIPT_ROOT:-}"; then + mkdir -p "$IB_HTML_PREPAINT_RECEIPT_ROOT" + cp "$output" "$IB_HTML_PREPAINT_RECEIPT_ROOT/$kind.prepaint" + cp "$manifest" "$IB_HTML_PREPAINT_RECEIPT_ROOT/$kind.admission.tsv" + cp "$work/$kind.receipt" "$IB_HTML_PREPAINT_RECEIPT_ROOT/$kind.conversion.tsv" + fi grep -Fx "$(printf 'conversion\tPASS')" "$work/$kind.receipt" printf 'fixture\t%s\tPASS\n' "$kind" } From d228f78b00e56a34a93f9f58ca908977ef1c1249 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:16:39 -0400 Subject: [PATCH 43/55] =?UTF-8?q?Pin=20independent=20AI-CI=20to=20inspect?= =?UTF-8?q?=20actual=20generated=20Idri=C3=A7=20wire=20and=20retain=20comp?= =?UTF-8?q?iler/source/output=20receipts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/idric-core.yml | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index c2b7235e..49a13b4e 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -113,11 +113,44 @@ jobs: /tmp/idric/bin/idris2 HtmlPrepaintCommand.idric -o ib-html-prepaint test -x build/exec/ib-html-prepaint - - name: Independent HTML prepaint wire and corruption-oracle acceptance + - name: Check out independently accepted AI-CI Prepaint consumer + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + repository: isomorphisms/ai-ci + ref: 0fb20839a2da68be0aafc10511e13c634bfa02b9 + path: .tools/ai-ci-prepaint + persist-credentials: false + + - name: Independently verify actual Idriç output and source-bound mutations + env: + IB_AICI_PREPAINT_VERIFIER: ${{ github.workspace }}/.tools/ai-ci-prepaint/ib-prepaint/verify.sh + IB_HTML_PREPAINT_RECEIPT_ROOT: ${{ runner.temp }}/ib-html-prepaint-receipts run: | + test "$(git -C .tools/ai-ci-prepaint rev-parse HEAD)" = 0fb20839a2da68be0aafc10511e13c634bfa02b9 sh -n bin/ib_emit_html_prepaint.grease + sh -n bin/ib_arena_to_html_prepaint.grease sh -n tests/test_html_prepaint.sh + sh .tools/ai-ci-prepaint/ib-prepaint/self-test.sh sh tests/test_html_prepaint.sh + { + printf 'repository\tisomorphisms/ib\n' + printf 'source_commit\t%s\n' "$(git rev-parse HEAD)" + printf 'idric_commit\t%s\n' "$IDRIC_REF" + printf 'independent_ai_ci_commit\t0fb20839a2da68be0aafc10511e13c634bfa02b9\n' + printf 'compiler_version\t%s\n' "$(/tmp/idric/bin/idris2 --version)" + printf 'platform\t%s\n' "$(uname -sm)" + } > "$IB_HTML_PREPAINT_RECEIPT_ROOT/inputs.tsv" + sha256sum src/build/exec/ib-html-prepaint src/build/exec/ib-html-prepaint-smoke \ + tests/prepaint_v1_oracle.c src/IB/HtmlPrepaint.idric src/IB/HtmlLinks.idric \ + > "$IB_HTML_PREPAINT_RECEIPT_ROOT/artifact-inputs.sha256" + (cd "$IB_HTML_PREPAINT_RECEIPT_ROOT" && sha256sum *.prepaint > wire.sha256) + + - name: Retain independently checked fixture receipts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ib-html-prepaint-idric-and-aici + path: ${{ runner.temp }}/ib-html-prepaint-receipts/ + if-no-files-found: error - name: Exercise browser-owned Idriç core run: sh bin/ci_browser_foundation.grease exercise-core From be86498f7860d03e96e3498fa7d0ab491cb08b5d Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:18:44 -0400 Subject: [PATCH 44/55] =?UTF-8?q?Record=20executed=20Idri=C3=A7-to-AICI=20?= =?UTF-8?q?success=20and=20preserve=20unobserved=20Arena/C67=20boundaries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/html-prepaint-admission.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/html-prepaint-admission.md b/docs/html-prepaint-admission.md index 118b0efd..08e74bec 100644 --- a/docs/html-prepaint-admission.md +++ b/docs/html-prepaint-admission.md @@ -1,8 +1,6 @@ # IB-C67 HTML → typed Prepaint Version 1 -Status: **DRAFT / NOT QUALIFIED** until an actual Idriç execution, independent -consumer and source admission have succeeded. This is the first complete-revision -conversion only. It is not a C67 URL-browser acceptance receipt. +Status: **host fixture converter QUALIFIED / retained Arena HTML BLOCKED / physical C67 NOT_RUN**. Executed [IB CI run 38653082213](https://github.com/isomorphisms/ib/actions/runs/38653082213) passed both `idric-core` and `scientific-media` on `d228f78b00e56a34a93f9f58ca908977ef1c1249`; the pinned separate `isomorphisms/ai-ci@0fb20839a2da68be0aafc10511e13c634bfa02b9` independently consumed the actual generated fixture bytes and ran its wrong-output self-tests. [Retained artifact 11935167962](https://github.com/isomorphisms/ib/actions/runs/38653082213/artifacts/11935167962) SHA-256: `acfe20c464eb4955a75fd5f27a31f038dcaa806c1936557f01dc3e1a5b20f9c6`. These are offline fixtures, **not live ICU network data or a C67 device receipt**. ## Owners and immutable starting points @@ -80,11 +78,11 @@ PR #152 must be reconciled before an Arena-owner receipt can be supplied. | Check | Authority | Current status | |---|---|---| -| Idriç typecheck + actual executable | IB CI | NOT_RUN until workflow completes | -| Offline example.com-shaped fixture | Idriç + independent C11 oracle and frozen golden | NOT_RUN | -| Offline arXiv-shaped fixture | Idriç + independent C11 oracle and frozen golden | NOT_RUN | -| Bad digest, generation, source ID, UTF-8 and omitted/stale metadata | Grease refusal, last-valid byte comparison | NOT_RUN | -| Mutated heading, link, dropped text and malformed escaping | Independent oracle/golden | NOT_RUN | +| Idriç typecheck + actual executable | IB CI #38653082213 | **PASS**, Linux/host fixtures | +| Offline example.com-shaped fixture | Actual Idriç output + independent C11/AI-CI oracle and frozen golden | **PASS** | +| Offline arXiv-shaped fixture | Actual Idriç output + independent C11/AI-CI oracle and frozen golden | **PASS** | +| Bad digest, generation, source ID, UTF-8 and incomplete admission | Grease refusal, last-valid byte comparison | **PASS**, fixture-only | +| Mutated heading, link, dropped text and malformed escaping | C11 oracle + independent AI-CI + golden | **PASS**, fixture-only | | Real ICU HTTPS → committed Arena → admitted metadata | ICU PR #35 + IB PR #152 | BLOCKED | | Binder/PFD delivers new projection to reader | IB PR #153 | NOT_RUN | | Physical C67 paints example.com and navigates links | User-observed device + receipts | NOT_RUN | From 564c9b3f076bd7eaf2daad66d3d5ac341e98f9a0 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:19:52 -0400 Subject: [PATCH 45/55] =?UTF-8?q?Exercise=20HTML=20base=20URL,=20query=20l?= =?UTF-8?q?inks,=20implied=20headings/paragraphs=20and=20list=20order=20in?= =?UTF-8?q?=20Idri=C3=A7=20smoke?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/HtmlPrepaintSmoke.idric | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/HtmlPrepaintSmoke.idric b/src/HtmlPrepaintSmoke.idric index cfdb420b..6eb5f41c 100644 --- a/src/HtmlPrepaintSmoke.idric +++ b/src/HtmlPrepaintSmoke.idric @@ -27,6 +27,18 @@ evidence = "https://example.com/docs/page" "https://example.com/docs/page" "text/html" "utf-8" "200" True True +base_and_omissions : String +base_and_omissions = + "" ++ + "

First

Second

A

BQuery" ++ + "

  • Alpha
  • Beta
" + +base_projection : String +base_projection = + case convert_admitted_html evidence base_and_omissions of + Left reason ⇒ "REFUSED:" ++ reason + Right produced ⇒ produced.wire + bad_evidence : HtmlEvidence bad_evidence = HtmlSourceEvidence "arena:fixture-example" "generation-1" @@ -86,6 +98,12 @@ main = Just "https://example.com/a?q=2") , ("unsafe-scheme", resolve_html_reference "https://example.com/" "javascript:bad" == Nothing) + , ("html-base-query", has "link\tQuery\thttps://example.org/docs/?q=3\n" base_projection) + , ("implied-heading-two", has "heading\t2\tFirst\n" base_projection) + , ("implied-heading-three", has "heading\t3\tSecond\n" base_projection) + , ("implied-paragraph", has "text\tA\n" base_projection && has "text\tB\n" base_projection) + , ("list-alpha", has "text\tAlpha\n" base_projection) + , ("list-beta", has "text\tBeta\n" base_projection) ] okay <- check tests if okay then pure () else exitFailure From 900a67868b8699c91f2e95fec3efa6a2947bf135 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:20:09 -0400 Subject: [PATCH 46/55] Prove unobserved PR152 Arena v1 cannot overwrite last valid HTML prepaint --- tests/test_arena_html_refusals.sh | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_arena_html_refusals.sh diff --git a/tests/test_arena_html_refusals.sh b/tests/test_arena_html_refusals.sh new file mode 100644 index 00000000..c0d4251e --- /dev/null +++ b/tests/test_arena_html_refusals.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# A v1 body-only PR #152 receipt must never become an admitted HTML page. +set -eu +root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +mkdir -p "$work/arena/receipts" +manifest="$work/arena/receipts/request-1.tsv" +cat > "$manifest" <<'EOF' +schema ib-icu-arena-v1 +request_id request-1 +requested_url https://example.com/ +resolved_url UNOBSERVED +http_status UNOBSERVED +mime_type UNOBSERVED +transport icu-app-process-via-binder-shizuku +arena_name responses.arena +arena_offset 0 +arena_end 2 +byte_count 2 +sha256 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +reopen_check MATCH +navigation_admitted NO +EOF +page="$work/display.prepaint" +cp "$root/tests/fixtures/ib-html/example.prepaint" "$page" +cp "$page" "$work/last-valid" +refuse() { + reason=$1 + if sh "$root/bin/ib_arena_to_html_prepaint.grease" \ + request-1 "$work/arena" generation-1 "$page" \ + > "$work/result" 2> "$work/error"; then + echo "unexpected admitted source: $reason" >&2 + exit 1 + fi + grep -F "$reason" "$work/error" + cmp -s "$page" "$work/last-valid" +} +refuse 'navigation not admitted' +sed 's/navigation_admitted.*$/navigation_admitted YES/' "$manifest" > "$work/replaced" +mv "$work/replaced" "$manifest" +refuse 'out-of-order or missing generation' +sed '/^mime_type/d' "$manifest" > "$work/replaced" +mv "$work/replaced" "$manifest" +refuse 'unadmitted content type' +printf 'arena-v1-unobserved-refused\tPASS\n' From a094cccbff25e27fdd7a55ed5c5693b4b293311d Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:20:18 -0400 Subject: [PATCH 47/55] Run explicit unobserved ICU/Arena receipt refusal and last-valid-page retention test --- .github/workflows/idric-core.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 49a13b4e..5b174e8e 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -129,6 +129,8 @@ jobs: test "$(git -C .tools/ai-ci-prepaint rev-parse HEAD)" = 0fb20839a2da68be0aafc10511e13c634bfa02b9 sh -n bin/ib_emit_html_prepaint.grease sh -n bin/ib_arena_to_html_prepaint.grease + sh -n tests/test_arena_html_refusals.sh + sh tests/test_arena_html_refusals.sh sh -n tests/test_html_prepaint.sh sh .tools/ai-ci-prepaint/ib-prepaint/self-test.sh sh tests/test_html_prepaint.sh From 316db73701e0686ccacb13e69c3b448874f0a639 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:20:52 -0400 Subject: [PATCH 48/55] Extend regression coverage for RTL, source size, charset and missing final URL --- src/HtmlPrepaintSmoke.idric | 1 + 1 file changed, 1 insertion(+) diff --git a/src/HtmlPrepaintSmoke.idric b/src/HtmlPrepaintSmoke.idric index 6eb5f41c..92c4bfab 100644 --- a/src/HtmlPrepaintSmoke.idric +++ b/src/HtmlPrepaintSmoke.idric @@ -86,6 +86,7 @@ main = , ("paragraph-before-link", has "text\tprefix\n" wire) , ("paragraph-after-link", has "text\tsuffix\n" wire) , ("non-bmp", has "🙂" wire) + , ("rtl-preserved", has "שלום" wire) , ("numeric-combining", has "é" wire) , ("table-row", has "row\tA\tB\n" wire) , ("unsafe-no-link", not (has "link\tUnsafe" wire)) From d207114a3520232f8d54c8cb90e60821ff8444d8 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:20:55 -0400 Subject: [PATCH 49/55] Extend regression coverage for RTL, source size, charset and missing final URL --- tests/test_html_prepaint.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_html_prepaint.sh b/tests/test_html_prepaint.sh index 4d58af83..6ac62906 100644 --- a/tests/test_html_prepaint.sh +++ b/tests/test_html_prepaint.sh @@ -98,6 +98,15 @@ export IB_EXPECTED_SOURCE_GENERATION sed 's@^mime_type.*@mime_type\tapplication/octet-stream@' "$work/good.tsv" > "$manifest" test_failure_preserves unsupported_mime cp "$work/good.tsv" "$manifest" +sed 's@^charset.*@charset\tiso-8859-1@' "$work/good.tsv" > "$manifest" +test_failure_preserves unsupported_charset +cp "$work/good.tsv" "$manifest" +sed 's@^byte_count.*@byte_count\t4194305@' "$work/good.tsv" > "$manifest" +test_failure_preserves oversized_body +cp "$work/good.tsv" "$manifest" +sed 's@^resolved_url.*@resolved_url\tUNOBSERVED@' "$work/good.tsv" > "$manifest" +test_failure_preserves unobserved_final_url +cp "$work/good.tsv" "$manifest" sed 's@^acquisition_complete.*@acquisition_complete\tNO@' "$work/good.tsv" > "$manifest" test_failure_preserves incomplete_source cp "$work/good.tsv" "$manifest" From 672dbcc9200d4efb75a18c5bbd7b1831925fed5e Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:21:23 -0400 Subject: [PATCH 50/55] Reach MIME refusal only after satisfying prior source-generation and completeness checks --- tests/test_arena_html_refusals.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_arena_html_refusals.sh b/tests/test_arena_html_refusals.sh index c0d4251e..fe295d17 100644 --- a/tests/test_arena_html_refusals.sh +++ b/tests/test_arena_html_refusals.sh @@ -40,7 +40,11 @@ refuse 'navigation not admitted' sed 's/navigation_admitted.*$/navigation_admitted YES/' "$manifest" > "$work/replaced" mv "$work/replaced" "$manifest" refuse 'out-of-order or missing generation' -sed '/^mime_type/d' "$manifest" > "$work/replaced" +{ + printf 'source_generation\tgeneration-1\n' + printf 'acquisition_complete\tYES\n' +} >> "$manifest" +sed 's@^mime_type.*@mime_type\tUNOBSERVED@' "$manifest" > "$work/replaced" mv "$work/replaced" "$manifest" refuse 'unadmitted content type' printf 'arena-v1-unobserved-refused\tPASS\n' From f992af5f15987a93c56850916f7e2b84d6ae57a6 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:24:03 -0400 Subject: [PATCH 51/55] =?UTF-8?q?Integrate=20unmodified=20pinned=20PR152?= =?UTF-8?q?=20Arena=20producer/reader=20into=20Idri=C3=A7=20V1=20using=20e?= =?UTF-8?q?xplicitly=20mocked=20ICU=20metadata?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_arena_v2_to_html_prepaint.sh | 141 ++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tests/test_arena_v2_to_html_prepaint.sh diff --git a/tests/test_arena_v2_to_html_prepaint.sh b/tests/test_arena_v2_to_html_prepaint.sh new file mode 100644 index 00000000..73cd7fb3 --- /dev/null +++ b/tests/test_arena_v2_to_html_prepaint.sh @@ -0,0 +1,141 @@ +#!/bin/sh +# Host-only integration against the EXACT external PR #152 ICU/Arena producer +# and reader with a marked MOCK ICU packet. No live network or Android device. +set -eu +root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +test -f "$root/.deps/ib-arena/bin/ib_android_icu_arena.grease" || { + echo 'pinned upstream Arena source unavailable' >&2; exit 2; +} +arena_tool=${IB_APPEND_ARENA:-} +slice_tool=${IB_ARENA_SLICE:-} +converter=${IB_HTML_PREPAINT_EXEC:-} +oracle=${IB_PREPAINT_ORACLE:-} +test -x "$arena_tool" && test -x "$slice_tool" && + test -x "$converter" && test -x "$oracle" || { + echo 'compiled Arena/Idriç/independent consumer executables required' >&2; exit 2; + } +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +mkdir -p "$work/lib" "$work/bin" "$work/private" "$work/output" +chmod 700 "$work/private" +cp "$root/.deps/ib-arena/bin/ib_android_icu_arena.grease" "$work/bin/" +cp "$root/.deps/ib-arena/bin/ib_android_arena_read.grease" "$work/bin/" +cp "$root/.deps/ib-arena/lib/android_shizuku_icu_provider.grease" "$work/lib/" +cp "$root/.deps/ib-arena/lib/android_binder_icu.grease" "$work/lib/" + +# This mock replaces *only* the Shizuku command invocation. Production +# ICU PR #35 capture, framing, validation, canonical Arena and source +# reopen logic remain unchanged, from exact upstream SHA. +cat > "$work/lib/android_shizuku_host.grease" <<'MOCK' +ib_shizuku_authority() { printf 'shell\n'; } +ib_shizuku_run_bounded() { + test "$1" = 30 || return 90 + case "$2" in *ICU_CAPTURE_BODY_PATH*ICU_CAPTURE_METADATA_PATH*Idric.Generated\ get*) ;; *) return 91 ;; esac + body=$IB_MOCK_SOURCE_BODY + test -f "$body" || return 92 + body_bytes=$(wc -c < "$body" | tr -d '[:space:]') + { + printf 'http_status\t200\n' + printf 'content_type\ttext/html; charset=utf-8\n' + printf 'content_encoding\tidentity\n' + printf 'declared_length\t%s\n' "$body_bytes" + printf 'received_length\t%s\n' "$body_bytes" + printf 'utf8\tvalid\n' + printf 'resolved_url\thttps://example.com/\n' + printf 'redirect_count\t0\n' + } > "$IB_MOCK_METADATA" + metadata_bytes=$(wc -c < "$IB_MOCK_METADATA" | tr -d '[:space:]') + printf 'IBICU01 %08d %08d\n' "$metadata_bytes" "$body_bytes" + cat "$IB_MOCK_METADATA" + cat "$body" +} +MOCK + +export IB_MOCK_SOURCE_BODY="$root/tests/fixtures/ib-html/example.html" +export IB_MOCK_METADATA="$work/mock-icu-metadata.tsv" +export IB_APPEND_ARENA="$arena_tool" +export IB_ARENA_SLICE="$slice_tool" +export IB_C67_ICU_DEX=/data/local/tmp/ib-icu/classes.dex +export IB_C67_ICU_SO=/data/local/tmp/ib-icu/libicu_transport.so +export IB_C67_ICU_DEX_SHA256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export IB_C67_ICU_SO_SHA256=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +export IB_HTML_PREPAINT_EXEC="$converter" +export IB_PREPAINT_ORACLE="$oracle" +"$arena_tool" create "$work/private/responses.arena" 16777216 >/dev/null +sh "$work/bin/ib_android_icu_arena.grease" host-example \ + https://example.com/ "$work/private" 17 > "$work/producer.tsv" +grep -Fx "$(printf 'committed\thost-example\t0\t%s\tYES' \ + "$(wc -c < "$IB_MOCK_SOURCE_BODY" | tr -d '[:space:]')")" "$work/producer.tsv" +receipt="$work/private/receipts/host-example.tsv" +test "$(awk -F '\t' '$1=="schema" {print $2}' "$receipt")" = ib-icu-arena-v2 +test "$(awk -F '\t' '$1=="source_generation" {print $2}' "$receipt")" = 17 +test "$(awk -F '\t' '$1=="navigation_admitted" {print $2}' "$receipt")" = YES + +# Host sh entrypoint for the unmodified upstream verified Arena reader. +cat > "$work/ib-arena-reader" <<'WRAPPER' +#!/bin/sh +exec sh "$IB_TEST_READER_SCRIPT" "$@" +WRAPPER +chmod 700 "$work/ib-arena-reader" +export IB_TEST_READER_SCRIPT="$work/bin/ib_android_arena_read.grease" +export IB_ARENA_READ_EXEC="$work/ib-arena-reader" +source_digest=$(sha256sum "$IB_MOCK_SOURCE_BODY" | cut -d ' ' -f 1) +source_length=$(wc -c < "$IB_MOCK_SOURCE_BODY" | tr -d '[:space:]') +sh "$IB_ARENA_READ_EXEC" host-example "$work/private" > "$work/reopened.html" +cmp "$work/reopened.html" "$IB_MOCK_SOURCE_BODY" + +projection="$work/output/host-example.prepaint" +sh "$root/bin/ib_arena_to_html_prepaint.grease" \ + host-example "$work/private" 17 "$projection" > "$work/conversion.tsv" +cmp "$root/tests/fixtures/ib-html/example.prepaint" "$projection" +"$IB_PREPAINT_ORACLE" "$projection" + +if test -n "${IB_AICI_PREPAINT_VERIFIER:-}"; then + { + printf 'source_key\tarena:host-example\n' + printf 'source_generation\t17\n' + printf 'source_sha256\t%s\n' "$source_digest" + printf 'source_bytes\t%s\n' "$source_length" + } > "$work/source-admission.tsv" + sh "$IB_AICI_PREPAINT_VERIFIER" "$projection" \ + "$root/tests/fixtures/ib-html/example.prepaint" \ + "$work/source-admission.tsv" "$work/reopened.html" arena:host-example 17 +fi + +cp "$projection" "$work/last-valid.prepaint" +cp "$receipt" "$work/valid-receipt.tsv" +refuse() { + case_name=$1 + if sh "$root/bin/ib_arena_to_html_prepaint.grease" \ + host-example "$work/private" 17 "$projection" \ + > "$work/refused.out" 2> "$work/refused.err"; then + echo "accepted mutated v2 Arena source: $case_name" >&2 + exit 1 + fi + cmp "$projection" "$work/last-valid.prepaint" + printf 'arena-v2-refusal\t%s\tPASS\n' "$case_name" +} +sed 's@^sha256.*@sha256\t0000000000000000000000000000000000000000000000000000000000000000@' \ + "$work/valid-receipt.tsv" > "$receipt" +refuse corrupted_arena_digest +sed 's@^source_generation.*@source_generation\t18@' "$work/valid-receipt.tsv" > "$receipt" +refuse stale_source_generation +sed 's@^navigation_admitted.*@navigation_admitted\tNO@' "$work/valid-receipt.tsv" > "$receipt" +refuse unadmitted_navigation +cp "$work/valid-receipt.tsv" "$receipt" + +if test -n "${IB_HTML_PREPAINT_RECEIPT_ROOT:-}"; then + mkdir -p "$IB_HTML_PREPAINT_RECEIPT_ROOT" + cp "$projection" "$IB_HTML_PREPAINT_RECEIPT_ROOT/arena-mocked.prepaint" + cp "$work/valid-receipt.tsv" "$IB_HTML_PREPAINT_RECEIPT_ROOT/arena-mocked-source.tsv" + cp "$work/conversion.tsv" "$IB_HTML_PREPAINT_RECEIPT_ROOT/arena-mocked-conversion.tsv" + { + printf 'source_kind\tMOCKED_ICU_V2_NOT_LIVE\n' + printf 'source_sha256\t%s\n' "$source_digest" + printf 'source_bytes\t%s\n' "$source_length" + printf 'source_generation\t17\n' + printf 'upstream_source_commit\t%s\n' "$(git -C "$root/.deps/ib-arena" rev-parse HEAD)" + } > "$IB_HTML_PREPAINT_RECEIPT_ROOT/arena-mocked-provenance.tsv" +fi +printf 'arena-v2-mock-through-actual-idric\tPASS\n' +printf 'real-icu-https\tNOT_RUN\nphysical-c67\tNOT_RUN\n' From 217b7fd181cfc07d2d5ea14186cd34e0faf0e2ea Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:24:31 -0400 Subject: [PATCH 52/55] =?UTF-8?q?Run=20actual=20pinned=20canonical=20Arena?= =?UTF-8?q?=20v2=20producer/readback=20into=20checked=20Idri=C3=A7=20and?= =?UTF-8?q?=20AICI=20with=20MOCK=20ICU=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/idric-core.yml | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 5b174e8e..154d0800 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -154,6 +154,62 @@ jobs: path: ${{ runner.temp }}/ib-html-prepaint-receipts/ if-no-files-found: error + - name: Check out verified Arena v2 acquisition and reader implementation + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: isomorphisms/ib + ref: e2a2a17a4378b7c3845ec43f42d1d3360b12f051 + path: .deps/ib-arena + persist-credentials: false + + - name: Check out canonical append-fat userspace Arena + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: isomorphisms/sd-card-append-fat + ref: 05a0cb7246c5a9f46abfbe92925812e7b607f454 + path: .deps/appendfat + persist-credentials: false + + - name: Resolve qualified host ICK compiler + uses: isomorphisms/ai-ci/ick-host@015cc7901ae0b3ad262b476f24e129b53c56db95 + id: ick + + - name: Reopen real canonical Arena bytes and compile with actual Idriç converter + env: + ICK: ${{ steps.ick.outputs.compiler }} + IB_AICI_PREPAINT_VERIFIER: ${{ github.workspace }}/.tools/ai-ci-prepaint/ib-prepaint/verify.sh + IB_HTML_PREPAINT_RECEIPT_ROOT: ${{ runner.temp }}/ib-html-prepaint-receipts + run: | + test "$(git -C .deps/ib-arena rev-parse HEAD)" = e2a2a17a4378b7c3845ec43f42d1d3360b12f051 + test "$(git -C .deps/appendfat rev-parse HEAD)" = 05a0cb7246c5a9f46abfbe92925812e7b607f454 + mkdir -p build/ib-html-arena + "$ICK" -fno-link-libatomic -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + .deps/appendfat/lib/appendfat_arena.c \ + .deps/appendfat/tools/appendfat_arena.c \ + -o build/ib-html-arena/appendfat_arena + cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + .deps/ib-arena/native/android/ib_arena_slice.c \ + -o build/ib-html-arena/ib-arena-slice + cc -std=c11 -O2 -Wall -Wextra -Wpedantic -Werror \ + tests/prepaint_v1_oracle.c -o build/ib-html-arena/prepaint-oracle + export IB_APPEND_ARENA="$GITHUB_WORKSPACE/build/ib-html-arena/appendfat_arena" + export IB_ARENA_SLICE="$GITHUB_WORKSPACE/build/ib-html-arena/ib-arena-slice" + export IB_HTML_PREPAINT_EXEC="$GITHUB_WORKSPACE/src/build/exec/ib-html-prepaint" + export IB_PREPAINT_ORACLE="$GITHUB_WORKSPACE/build/ib-html-arena/prepaint-oracle" + sh -n tests/test_arena_v2_to_html_prepaint.sh + sh tests/test_arena_v2_to_html_prepaint.sh + sha256sum build/ib-html-arena/appendfat_arena build/ib-html-arena/ib-arena-slice \ + build/ib-html-arena/prepaint-oracle \ + > "$IB_HTML_PREPAINT_RECEIPT_ROOT/arena-host-executables.sha256" + (cd "$IB_HTML_PREPAINT_RECEIPT_ROOT" && sha256sum *.prepaint > wire.sha256) + + - name: Retain qualified Arena-v2 mocked-origin integration evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ib-html-prepaint-arena-v2-mock + path: ${{ runner.temp }}/ib-html-prepaint-receipts/ + if-no-files-found: error + - name: Exercise browser-owned Idriç core run: sh bin/ci_browser_foundation.grease exercise-core From 33191f743e2f3bc51ba950f499187db595d43ec2 Mon Sep 17 00:00:00 2001 From: i Date: Wed, 14 Oct 2026 19:25:12 -0400 Subject: [PATCH 53/55] Reconcile newly implemented ICU metadata-backed Arena V2 dependency without claiming live phone acceptance --- docs/html-prepaint-admission.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/html-prepaint-admission.md b/docs/html-prepaint-admission.md index 08e74bec..e3732940 100644 --- a/docs/html-prepaint-admission.md +++ b/docs/html-prepaint-admission.md @@ -7,12 +7,19 @@ Status: **host fixture converter QUALIFIED / retained Arena HTML BLOCKED / physi - Semantic base: PR #148, `work/linux-shared-contract`, originally `2425d21d984de21e5b65ae0d2e09c586120239ee`. This includes the types and existing serializer from #48/#123. -- Upstream Arena producer: PR #152, original head - `cc5f64b84c316b88d4c5c67fe4ea7ee5fa1b9bbe`. - `bin/ib_android_arena_read.grease` supplies a digest-checked range but its - receipt has `resolved_url=UNOBSERVED`, `mime_type=UNOBSERVED`, - `http_status=UNOBSERVED`, `navigation_admitted=NO`, and no pinned source - generation or charset. **No live HTML can pass source admission yet.** +- Upstream Arena producer and verified source reader: PR #152, whose + Version 2 candidate was pinned at + `e2a2a17a4378b7c3845ec43f42d1d3360b12f051`. + It now consumes ICU #35's *paired same-request* response body and actual + metadata, validates source key/generation, HTTP status, MIME/charset, + response completeness, final URL and SHA-256, and independently reopens the + committed canonical Arena range. Its legacy v1 body-only receipt remains + permanently navigation-denied. Its v2 host test passed at + [run 38652958137](https://github.com/isomorphisms/ib/actions/runs/38652958137), + **with mocked Shizuku/ICU transport**, not actual C67 HTTPS. The separate + converter now has a pinned cross-branch mock-integration test for the same + v2 producer/reader and the actual Idriç command; that test is not yet a + live device acceptance claim. - Separate reader-handoff owner: PR #153, original head `9cf28c2f6c38cb12bb906a844e061ebfab34f943`. - C67 native Material reader: PR #129. It currently paints a pinned snapshot. @@ -53,8 +60,9 @@ receipt and an **already verified** body path, then requires an exact expected source key and `IB_EXPECTED_SOURCE_GENERATION`. The receipt is authority only when supplied by the trusted source-store owner, not by an app/remote page. The `fixture` kind is prohibited unless `IB_ALLOW_FIXTURE=1`; tests always -label fixtures. This adapter does **not** grant Arena metadata qualification: -PR #152 must be reconciled before an Arena-owner receipt can be supplied. +label fixtures. For a v2 Arena source, this adapter passes through the +**unchanged pinned** #152 producer/verified reader. Mocked metadata proves only +contract compatibility, never live acquisition or Android reader delivery. ## Supported now / explicit limitations @@ -83,7 +91,8 @@ PR #152 must be reconciled before an Arena-owner receipt can be supplied. | Offline arXiv-shaped fixture | Actual Idriç output + independent C11/AI-CI oracle and frozen golden | **PASS** | | Bad digest, generation, source ID, UTF-8 and incomplete admission | Grease refusal, last-valid byte comparison | **PASS**, fixture-only | | Mutated heading, link, dropped text and malformed escaping | C11 oracle + independent AI-CI + golden | **PASS**, fixture-only | -| Real ICU HTTPS → committed Arena → admitted metadata | ICU PR #35 + IB PR #152 | BLOCKED | +| Mock ICU metadata → actual canonical Arena v2 → Idriç V1 | Pinned PR #152 + canonical appendfat + AICI | COMMITTED / exact-head CI queued | +| Real ICU HTTPS → committed Arena → admitted metadata | ICU PR #35 + IB PR #152 | NOT_RUN on physical C67 | | Binder/PFD delivers new projection to reader | IB PR #153 | NOT_RUN | | Physical C67 paints example.com and navigates links | User-observed device + receipts | NOT_RUN | | ARM32 A1/C67, ARM64 C67 converter execution | Cat Food/CI evidence | NOT_RUN | From ff8423697ef1d25140f4e77594b224ce7b87fef7 Mon Sep 17 00:00:00 2001 From: i Date: Sat, 24 Oct 2026 22:48:33 -0400 Subject: [PATCH 54/55] Connect typed Prepaint to existing source-bound Unicode reflow Reuse unchanged reflow modules from 4a7380c5f25409443c5e8c1ae17bd358372012fc. Preserve canonical revisions, original blocks and HTML admission sidecar; derive per-block codepoint spans with width and measurement identity. Independent Idric host checks exercise mixed-block/source reconstruction and reject a compiled block-order mutant. Source-bound streaming, typographic shaping and Android integration remain separate. Tested tree 9d61c0e5f21ef0246a808d786c9298f1ab590d55. --- .../PrepaintReflowChecks.idric | 125 +++++++++ .../PrepaintReflowMoonChecks.idric | 134 ++++++++++ .../prepaint-source-reflow/attempt.md | 132 ++++++++++ src/IB/Prepaint/Reflow.idric | 79 ++++++ src/IB/Reflow.idric | 85 +++++++ src/IB/Reflow/Arithmetic.idric | 31 +++ src/IB/Reflow/Balanced.idric | 64 +++++ src/IB/Reflow/Greedy.idric | 21 ++ src/IB/Reflow/Incremental.idric | 61 +++++ src/IB/Reflow/KnuthPlass.idric | 237 ++++++++++++++++++ src/IB/Reflow/Objectives.idric | 80 ++++++ src/IB/Reflow/Source.idric | 221 ++++++++++++++++ src/IB/Reflow/SourceBreaks.idric | 201 +++++++++++++++ src/IB/Reflow/Streaming.idric | 101 ++++++++ src/IB/Reflow/Threshold.idric | 27 ++ src/ReflowSmoke.idric | 187 ++++++++++++++ src/SourceReflowSmoke.idric | 114 +++++++++ 17 files changed, 1900 insertions(+) create mode 100644 examples/autogenerated/prepaint-source-reflow/PrepaintReflowChecks.idric create mode 100644 examples/autogenerated/prepaint-source-reflow/PrepaintReflowMoonChecks.idric create mode 100644 examples/autogenerated/prepaint-source-reflow/attempt.md create mode 100644 src/IB/Prepaint/Reflow.idric create mode 100644 src/IB/Reflow.idric create mode 100644 src/IB/Reflow/Arithmetic.idric create mode 100644 src/IB/Reflow/Balanced.idric create mode 100644 src/IB/Reflow/Greedy.idric create mode 100644 src/IB/Reflow/Incremental.idric create mode 100644 src/IB/Reflow/KnuthPlass.idric create mode 100644 src/IB/Reflow/Objectives.idric create mode 100644 src/IB/Reflow/Source.idric create mode 100644 src/IB/Reflow/SourceBreaks.idric create mode 100644 src/IB/Reflow/Streaming.idric create mode 100644 src/IB/Reflow/Threshold.idric create mode 100644 src/ReflowSmoke.idric create mode 100644 src/SourceReflowSmoke.idric diff --git a/examples/autogenerated/prepaint-source-reflow/PrepaintReflowChecks.idric b/examples/autogenerated/prepaint-source-reflow/PrepaintReflowChecks.idric new file mode 100644 index 00000000..f3d0d898 --- /dev/null +++ b/examples/autogenerated/prepaint-source-reflow/PrepaintReflowChecks.idric @@ -0,0 +1,125 @@ +module PrepaintReflowChecks + +import Data.Text +import Data.Maybe +import IB.Strand +import IB.Prepaint +import IB.Prepaint.Version1 +import IB.HtmlPrepaint +import IB.Prepaint.Reflow +import IB.Reflow +import IB.Reflow.SourceBreaks +import System + +codepoint_width : Text → Number +codepoint_width text = length (unpack text) + +double_width : Text → Number +double_width text = 2 × codepoint_width text + +visible_lines : List SourceLine → List Text +visible_lines [] = [] +visible_lines (line :: rest) = line.visible_text :: visible_lines rest + +reconstruct : Text → List SourceLine → Text +reconstruct source [] = "" +reconstruct source (line :: rest) = + source_for_line source line ++ reconstruct source rest + +all_blocks_reconstruct : List BlockReflow → Bool +all_blocks_reconstruct [] = True +all_blocks_reconstruct (plan :: rest) = + (case plan.decoded_text of + Nothing ⇒ null plan.text_lines + Just text ⇒ reconstruct text plan.text_lines ≟ text) && + all_blocks_reconstruct rest + +positions_match : Number → List BlockReflow → Bool +positions_match position [] = True +positions_match position (plan :: rest) = + plan.block_position ≟ position && positions_match (S position) rest + +same_original_blocks : List InformationBlock → List BlockReflow → Bool +same_original_blocks [] [] = True +same_original_blocks (block :: rest) (plan :: plans) = + same_block block plan.original_block && same_original_blocks rest plans + where + same_block : InformationBlock → InformationBlock → Bool + same_block (Heading left_level left) (Heading right_level right) = + heading_level_number left_level ≟ heading_level_number right_level && left ≟ right + same_block (TextBlock left) (TextBlock right) = left ≟ right + same_block (Link left left_ref) (Link right right_ref) = + left ≟ right && link_reference_value left_ref ≟ link_reference_value right_ref + same_block (TableRow left) (TableRow right) = + table_row_cells left ≟ table_row_cells right + same_block _ _ = False +same_original_blocks _ _ = False + +source_identity : PrepaintRevision → Text +source_identity revision = nonempty_text_value revision.source.reference + +cases : HtmlPrepaintResult → Bool +cases original = + case (reflow_html_prepaint (Requirements 4 promptly) "codepoints-v1" codepoint_width original, + reflow_html_prepaint (Requirements 4 promptly) "double-codepoints-v1" double_width original) of + (Right result, Right wider) ⇒ + let layout = result.revision_layout in + let wider_layout = wider.revision_layout in + let retained = layout.source_revision in + let same_source = source_identity retained ≟ source_identity original.revision && + source_identity retained ≟ "arena-source;generation=generation-7;sha256=" ++ + original.evidence.claimed_sha256 in + let same_revision = revision_sequence_value retained.sequence ≟ + revision_sequence_value original.revision.sequence in + let same_wire = serialize_version1 [retained] ≟ Just original.wire in + let source_sidecar = result.source_result.evidence.source_generation ≟ "generation-7" && + result.source_result.evidence.claimed_sha256 ≟ original.evidence.claimed_sha256 in + let same_blocks = same_original_blocks original.revision.blocks layout.block_plans in + let widths = case (layout.block_plans, wider_layout.block_plans) of + (heading :: paragraph :: link :: row :: [], wide_heading :: _) ⇒ + visible_lines heading.text_lines ≟ ["漢字漢字"] && + visible_lines wide_heading.text_lines ≟ ["漢字", "漢字"] && + visible_lines paragraph.text_lines ≟ ["red", "blue"] && + visible_lines link.text_lines ≟ ["next"] && + isNothing row.decoded_text && null row.text_lines + _ ⇒ False in + same_source && same_revision && same_wire && source_sidecar && same_blocks && widths && + positions_match 0 layout.block_plans && all_blocks_reconstruct layout.block_plans && + layout.measurement_identity ≟ "codepoints-v1" && + wider_layout.measurement_identity ≟ "double-codepoints-v1" + _ ⇒ False + +refused : Either Text PrepaintLayout → Bool +refused (Left _) = True +refused _ = False + +refusal_cases : PrepaintRevision → Bool +refusal_cases revision = + refused (reflow_prepaint (Requirements 0 promptly) "metric" codepoint_width revision) && + refused (reflow_prepaint (Requirements 4 promptly) " " codepoint_width revision) && + refused (reflow_prepaint (Requirements 4 streamingly) "metric" codepoint_width revision) && + refused (reflow_prepaint (Requirements 4 typographically) "metric" codepoint_width revision) + +nontext_refusal_cases : PrepaintRevision → Bool +nontext_refusal_cases revision = + let empty_revision = record { blocks = [] } revision in + refused (reflow_prepaint (Requirements 4 streamingly) "metric" codepoint_width empty_revision) && + refused (reflow_prepaint (Requirements 4 typographically) "metric" codepoint_width empty_revision) && + refused (reflow_prepaint (Requirements 0 promptly) "metric" codepoint_width empty_revision) + +main : IO () +main = do + let digest = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + let evidence = HtmlSourceEvidence "arena-source" "generation-7" digest digest + 123 123 "https://example.test/request" "https://example.test/resolved" + "text/html" "utf-8" "200" True True + let html = "

漢字漢字

red blue

next
leftright
" + case convert_admitted_html evidence html of + Left problem ⇒ do + putStrLn ("prepaint-reflow-conversion=" ++ problem) + exitFailure + Right result ⇒ do + let passed = cases result && refusal_cases result.revision && + nontext_refusal_cases result.revision + putStrLn ("prepaint-source-reflow-cases=" ++ show passed) + if passed then pure () else exitFailure diff --git a/examples/autogenerated/prepaint-source-reflow/PrepaintReflowMoonChecks.idric b/examples/autogenerated/prepaint-source-reflow/PrepaintReflowMoonChecks.idric new file mode 100644 index 00000000..94ffc3d0 --- /dev/null +++ b/examples/autogenerated/prepaint-source-reflow/PrepaintReflowMoonChecks.idric @@ -0,0 +1,134 @@ +module PrepaintReflowMoonChecks + +import Data.Text +import Data.Maybe +import IB.Strand +import IB.Prepaint +import IB.Prepaint.Version1 +import IB.Prepaint.Reflow +import IB.Reflow +import IB.Reflow.Source +import IB.Reflow.SourceBreaks +import System + +measure_codepoints : Text → Number +measure_codepoints text = length (unpack text) + +source_name : PrepaintRevision → Text +source_name revision = nonempty_text_value revision.source.reference + +original_blocks : List BlockReflow → List InformationBlock +original_blocks plans = map (\plan ⇒ plan.original_block) plans + +same_origin : RepresentationOrigin → RepresentationOrigin → Bool +same_origin HtmlSource HtmlSource = True +same_origin PdfSource PdfSource = True +same_origin PlainTextSource PlainTextSource = True +same_origin DerivedSource DerivedSource = True +same_origin OtherSource OtherSource = True +same_origin _ _ = False + +same_completeness : RepresentationCompleteness → RepresentationCompleteness → Bool +same_completeness PartialSource PartialSource = True +same_completeness CompleteSource CompleteSource = True +same_completeness _ _ = False + +same_revision : PrepaintRevision → PrepaintRevision → Bool +same_revision expected actual = + source_name expected ≟ source_name actual && + same_origin expected.source.origin actual.source.origin && + same_completeness expected.source.completeness actual.source.completeness && + serialize_version1 [expected] ≟ serialize_version1 [actual] + +contiguous : Number → Number → List SourceLine → Bool +contiguous source_length offset [] = offset ≟ source_length +contiguous source_length offset (line :: rest) = + line.first_codepoint ≟ offset && + line.after_last_codepoint > offset && + line.after_last_codepoint <= source_length && + contiguous source_length line.after_last_codepoint rest + +-- Intentionally independent of the production source_for_line helper. +recover_slices : Text → List SourceLine → Text +recover_slices source [] = "" +recover_slices source (line :: rest) = + let before_end = take line.after_last_codepoint (unpack source) in + pack (drop line.first_codepoint before_end) ++ recover_slices source rest + +text_plan_valid : BlockReflow → Bool +text_plan_valid plan = case plan.decoded_text of + Nothing ⇒ null plan.text_lines + Just text ⇒ contiguous (measure_codepoints text) 0 plan.text_lines && + recover_slices text plan.text_lines ≟ text + +positions_valid : Number → List BlockReflow → Bool +positions_valid offset [] = True +positions_valid offset (plan :: rest) = + plan.block_position ≟ offset && positions_valid (S offset) rest + +layout_valid : PrepaintRevision → PrepaintLayout → Bool +layout_valid expected layout = + same_revision expected layout.source_revision && + same_revision expected (record { blocks = original_blocks layout.block_plans } layout.source_revision) && + positions_valid 0 layout.block_plans && all text_plan_valid layout.block_plans + +visible : List SourceLine → List Text +visible lines = map (\line ⇒ line.visible_text) lines + +mixed_blocks : Maybe (List InformationBlock) +mixed_blocks = do + link ← link_reference "https://example.test/next" + form ← form_reference "https://example.test/submit" + image ← image_reference "arena-image-id" + pure [Heading Heading2 "漢字漢字", TextBlock "a\r\nb", Link "ab­cd" link, + TableRow (table_row "first cell" ["second cell"]), + Form "form label" form, Image image "alternate" "caption" (Just link), + TextBlock "ab​cd", TextBlock "a b", TextBlock "éx", TextBlock "👩‍💻x"] + +check_revision : PrepaintRevision → Bool +check_revision revision = + case (reflow_prepaint (Requirements 3 promptly) "cp-v1" measure_codepoints revision, + reflow_prepaint (Requirements 4 evenly) "cp-v2" measure_codepoints revision) of + (Right layout, Right changed) ⇒ + let plans = layout.block_plans in + let same_mixed = layout_valid revision layout && layout_valid revision changed in + let geometry = case plans of + heading :: hard :: soft :: table :: form :: image :: zero :: no_break :: accent :: emoji :: [] ⇒ + visible heading.text_lines ≟ ["漢字漢", "字"] && + visible hard.text_lines ≟ ["a", "b"] && + visible soft.text_lines ≟ ["ab-", "cd"] && + isNothing table.decoded_text && isNothing form.decoded_text && isNothing image.decoded_text && + visible zero.text_lines ≟ ["ab", "cd"] && + visible no_break.text_lines ≟ ["a b"] && + visible accent.text_lines ≟ ["éx"] && + visible emoji.text_lines ≟ ["👩‍💻x"] + _ ⇒ False in + let dropped = record { block_plans = drop 1 plans } layout in + let reordered = record { block_plans = reverse plans } layout in + let new_revision = record { sequence = next_revision revision.sequence } revision in + let changed_source = case nonempty_text "another-arena-source" of + Nothing ⇒ revision + Just reference ⇒ record { source = record { reference = reference } revision.source } revision in + same_mixed && geometry && + not (layout_valid revision dropped) && not (layout_valid revision reordered) && + not (layout_valid new_revision layout) && not (layout_valid changed_source layout) && + layout.measurement_identity ≟ "cp-v1" && changed.measurement_identity ≟ "cp-v2" && + (case (layout.requirements, changed.requirements) of + (Requirements 3 promptly, Requirements 4 evenly) ⇒ True + _ ⇒ False) + _ ⇒ False + +main : IO () +main = do + let source = do + reference ← nonempty_text "arena-source;generation=7;sha256=fixture" + requested ← requested_address "https://example.test/requested" + resolved ← resolved_address "https://example.test/resolved" + blocks ← mixed_blocks + pure (Prepaint first_revision CompleteRevision + (Source reference HtmlSource CompleteSource) requested resolved (Just "title") blocks) + let passed = case source of + Nothing ⇒ False + Just revision ⇒ check_revision revision + putStrLn ("prepaint-moon-independent-cases=" ++ show passed) + if passed then pure () else exitFailure diff --git a/examples/autogenerated/prepaint-source-reflow/attempt.md b/examples/autogenerated/prepaint-source-reflow/attempt.md new file mode 100644 index 00000000..60b2fd81 --- /dev/null +++ b/examples/autogenerated/prepaint-source-reflow/attempt.md @@ -0,0 +1,132 @@ +# Typed Prepaint to source-bound reflow + +## Purpose and source boundaries + +Join existing typed Prepaint revisions to the existing Unicode source-bound +reflow implementation. This is derived layout; the revision and each original +information block remain unchanged. A block position identifies an occurrence +only within the retained immutable revision, not across different revisions. + +The base is IB HTML-to-Prepaint commit +`33191f743e2f3bc51ba950f499187db595d43ec2`. Reflow modules and their existing +smokes are reused unchanged from commit +`4a7380c5f25409443c5e8c1ae17bd358372012fc`. No line-breaking algorithm is added. + +## Values, operations, and invariants + +- Input: an existing `PrepaintRevision`, `ReflowRequirements`, a nonempty + measurement identity, and its `Text → Number` width callback. +- Output: either a refusal or a derived layout retaining that exact revision, + requirement, measurement identity, original blocks and block positions. +- Text headings, text paragraphs and link labels receive per-block + `SourceLine` plans. Table rows, forms and images remain explicitly unreflowed; + their semantic payloads are not flattened into invented text. +- Offsets count decoded Unicode code points in each block's text. They are not + original HTML byte offsets, UTF-16 offsets, glyph positions or pixels. +- The HTML-result entrypoint retains the complete `HtmlPrepaintResult`, + including `HtmlEvidence`. Version 1 wire bytes alone do not authenticate the + Arena source generation or digest. +- Width zero, absent measurement identity and unsupported source-bound + streaming/typographic modes must refuse. Existing no-split and source + reconstruction rules remain authoritative. +- All operations are pure. No I/O, networking, storage, native shaping, device + installation or renderer claim is introduced. + +## Intended signatures + +`reflow_prepaint : ReflowRequirements → Text → (Text → Number) → +PrepaintRevision → Either Text PrepaintLayout` + +`reflow_html_prepaint : ReflowRequirements → Text → (Text → Number) → +HtmlPrepaintResult → Either Text HtmlPrepaintLayout` + +## Compiler and verification + +The available Linux x86-64 compiler bundle identifies Idriç +`94dfd99bd3e376507fedc8611053b7173b2519f0`, Chez 10.4.1, producer run +`38090627721`, attempt 1. Use its actual `bin/idric` entrypoint and preserve +the first compiler failure rather than substituting another runtime. + +Verification must execute the real typed HTML converter and adapter, compare +known Unicode/width line plans, reconstruct exact decoded text through the +source spans, retain wire/source/revision/block identity, and exercise refusals. +Source-only or host execution does not prove native rendering or physical C67. + +## Result + +Implemented `IB.Prepaint.Reflow`. The adapter never reparses V1 wire output. +It retains the input revision and original blocks, records the measurement +identity, and delegates break selection to the unchanged source-bound solver. + +Actual host verification used a scratch copy of `src/` at +`/workspace/scratch/22d72635f498/u3-source-check`, with the canonical +`PrepaintReflowChecks.idric` copied into that source root. The compiler entrypoint +was `/workspace/scratch/58fb4da72caa/tools/idric/idric-linux-x86_64-94dfd99bd3e376507fedc8611053b7173b2519f0/bin/idric`; +its arguments were `PrepaintReflowChecks.idric -o prepaint-reflow-checks`. +Then the generated `build/exec/prepaint-reflow-checks` +was executed. It printed `prepaint-source-reflow-cases=True` and exited 0. +The existing `SourceReflowSmoke.idric` was separately compiled and executed; +it printed `source-reflow-unicode-offset-cases=True` and exited 0. +`sh tests/idric-public-vocabulary.sh` passed. + +The first check compile exposed a missing `Data.Maybe` import in the new +check program; adding the actual library import fixed it. No language or +runtime fallback was used. These are Chez host semantic results, not DEX, +Android layout, pixel/shaping or physical-device results. +The check supplies an admission-evidence fixture to the real typed converter; +it does not itself reopen or authenticate an Arena range. + +## Remaining language or integration work + +This slice does not make the whole-document HTML ingestion path bounded, remove +its existing 4 MiB admission ceiling, implement source-bound streaming or +typographic shaping, serialize source evidence into V1, or connect Android views. +The first integration boundary is the native view consumer supplying its real +measurement callback plus versioned measurement identity and consuming these +per-block plans. Acceptance must independently compare its copied/source text +and actual measured line geometry on the identified target. + +## Independent Moon checks + +The independent check program consumes the existing immutable revision and +derived layout. Its input domain includes ordered mixed semantic blocks, +Unicode decoded text, source/revision identities, line-codepoint intervals, +measurement identities and explicit reflow requirements. Its output is a +Boolean acceptance result with a failing process status for any mismatch. +It reconstructs decoded source directly with `take`/`drop` over Unicode code +points, independently of `source_for_line`. It compares serialized original +blocks and explicit source identity separately because V1 does not serialize +the source reference. It must reject a dropped/reordered block or changed +source/revision/metric, and preserve table/form/image payloads without +claiming those payloads received a text layout. Its only effects are checked +host compilation, output and exit status; it neither authenticates Arena +bytes nor performs native font measurement. + +Moon verified exact Earth commit +`d3d6c710a9b29ea501eb12fca97888804c515cd2` in a separate worktree. +The reused reflow paths match their declared `4a7380c5...` source exactly. +The compiler bundle's complete `payload.sha256` verification exited 0 and +`receipts/identity.txt` identifies `94dfd99b...`, Linux x86-64, Chez 10.4.1, +producer run 38090627721 attempt 1. + +In a fresh staged source directory, the exact `bin/idric` entrypoint compiled +`PrepaintReflowChecks.idric`, `PrepaintReflowMoonChecks.idric`, and +`SourceReflowSmoke.idric`. Each generated executable was launched through the +same bundle's absolute `bin/idric-env` entrypoint. The outputs were respectively +`prepaint-source-reflow-cases=True`, `prepaint-moon-independent-cases=True`, and +`source-reflow-unicode-offset-cases=True`, all with exit 0. The independent +program covers mixed heading/text/link/table/form/image blocks, exact Unicode +slice coverage, Han, CRLF, SHY, ZWSP, NBSP, combining marks, an overwide emoji +ZWJ sequence, source/revision changes and width/manner/measurement records. + +A separate scratch production mutant changed the final adapter construction +to use `reverse prepared`. The same independent check compiled successfully, +printed `prepaint-moon-independent-cases=False` and exited 1. The maintained +source was unchanged. The program also rejects dropped or reordered plans +and a stale source or revision directly. Public vocabulary and whitespace +checks passed. The first new fixture compile rejected the reserved word +`total` as a parameter name; renaming that fixture parameter fixed it. + +These are independently reproduced host semantic results. They do not qualify +bounded ingestion, native font measurement, Android/ART, Arena authentication, +or physical C67/A1 execution. diff --git a/src/IB/Prepaint/Reflow.idric b/src/IB/Prepaint/Reflow.idric new file mode 100644 index 00000000..fc4483f7 --- /dev/null +++ b/src/IB/Prepaint/Reflow.idric @@ -0,0 +1,79 @@ +module IB.Prepaint.Reflow + +import Data.Text +import IB.Strand +import IB.Prepaint +import IB.HtmlPrepaint +import IB.Reflow +import IB.Reflow.Source +import IB.Reflow.SourceBreaks + +-- This is a derived plan over an immutable revision. The position identifies +-- a block occurrence only within that retained revision. It is not a new +-- durable block identifier or an anchor that survives a changed revision. +public export +record BlockReflow where + constructor PreparedBlock + block_position : Number + original_block : InformationBlock + decoded_text : Maybe Text + text_lines : List SourceLine + +public export +record PrepaintLayout where + constructor PreparedRevision + source_revision : PrepaintRevision + requirements : ReflowRequirements + measurement_identity : Text + block_plans : List BlockReflow + +-- Keep the admission evidence, recovered log and semantic result alongside +-- their layout. V1 wire bytes alone do not authenticate the source range. +public export +record HtmlPrepaintLayout where + constructor PreparedHtml + source_result : HtmlPrepaintResult + revision_layout : PrepaintLayout + +block_text : InformationBlock → Maybe Text +block_text (Heading _ text) = Just text +block_text (TextBlock text) = Just text +block_text (Link text _) = Just text +block_text _ = Nothing + +prepare_blocks : ReflowRequirements → (Text → Number) → Number → + List InformationBlock → Either Text (List BlockReflow) +prepare_blocks request measure position [] = Right [] +prepare_blocks request measure position (block :: rest) = + case block_text block of + Nothing ⇒ do + remaining ← prepare_blocks request measure (S position) rest + Right (PreparedBlock position block Nothing [] :: remaining) + Just text ⇒ do + lines ← reflow_source request measure text + remaining ← prepare_blocks request measure (S position) rest + Right (PreparedBlock position block (Just text) lines :: remaining) + +-- The width callback and its nonempty identity describe the same measurement +-- provider. Offsets belong to decoded text in each block, never HTML bytes, +-- glyphs or UTF-16 units. No native shaping is implied by a Number result. +-- Unsupported manners refuse even when the revision has no textual blocks. +public export +reflow_prepaint : ReflowRequirements → Text → (Text → Number) → + PrepaintRevision → Either Text PrepaintLayout +reflow_prepaint request@(Requirements width manner) metric measure revision = + if trim metric ≟ "" then Left "Missing reflow measurement identity" + else case manner of + streamingly ⇒ Left "Source-bound streaming reflow is unavailable" + typographically ⇒ Left "Source-bound typographic reflow is unavailable" + _ ⇒ if width ≟ 0 then Left "Reflow width must be positive" + else do + prepared ← prepare_blocks request measure 0 revision.blocks + Right (PreparedRevision revision request metric prepared) + +public export +reflow_html_prepaint : ReflowRequirements → Text → (Text → Number) → + HtmlPrepaintResult → Either Text HtmlPrepaintLayout +reflow_html_prepaint request metric measure result = do + layout ← reflow_prepaint request metric measure result.revision + Right (PreparedHtml result layout) diff --git a/src/IB/Reflow.idric b/src/IB/Reflow.idric new file mode 100644 index 00000000..f75c38ec --- /dev/null +++ b/src/IB/Reflow.idric @@ -0,0 +1,85 @@ +module IB.Reflow + +import Data.Text +import IB.Reflow.Greedy +import IB.Reflow.Streaming +import IB.Reflow.Balanced +import IB.Reflow.Objectives +import IB.Reflow.Threshold +import IB.Reflow.KnuthPlass + +-- One verb; adverbs constrain its manner. The algorithm names occur only +-- below this selector. Each choice makes a documented tradeoff. +-- This initial selector makes an explicit *request* decision, not an unproven +-- ARM32/ARM64/x86-64 benchmark-based choice. +public export +choice reflow_adverb one_of + promptly + streamingly + evenly + minimizing_raggedness + ignoring_final_line + minimizing_worst_gap + without_extra_lines + typographically + +public export +choice reflow_way one_of + greedy + streaming_greedy + balanced + cubic + final_line_free + minimax + threshold_balanced + knuth_plass + +public export +record ReflowRequirements where + constructor Requirements + available_width : Number + how : reflow_adverb + +public export +select_reflow : ReflowRequirements → reflow_way +select_reflow (Requirements _ promptly) = greedy +select_reflow (Requirements _ streamingly) = streaming_greedy +select_reflow (Requirements _ evenly) = balanced +select_reflow (Requirements _ minimizing_raggedness) = cubic +select_reflow (Requirements _ ignoring_final_line) = final_line_free +select_reflow (Requirements _ minimizing_worst_gap) = minimax +select_reflow (Requirements _ without_extra_lines) = threshold_balanced +select_reflow (Requirements _ typographically) = knuth_plass + +-- A caller can alternatively use IB.Reflow.Streaming.feed_chunk, beginning +-- with begin_stream and ending with finish_stream or end_paragraph, to consume +-- raw decoded text chunks without constructing an entire paragraph. +-- +-- Contract for this shared batch surface: +-- * segments are atomic and in source order; +-- * neighboring segments on a line are joined with one display space; +-- * never change, drop, clip, or split a segment; +-- * an individually overwide segment remains overwide on its own line; +-- * the supplied width function determines whether a candidate line fits. +-- +-- All methods share that contract, not necessarily their break positions. +-- Complex-script shaping, Unicode line-break opportunities, source offsets, +-- hyphenation, and CSS-like justification remain separate from this contract. +public export +reflow : ReflowRequirements → (Text → Number) → List Text → + Either Text (List Text) +reflow (Requirements Z _) measure segments = + Left "Reflow width must be positive" +reflow request@(Requirements width _) measure segments = + case select_reflow request of + greedy ⇒ Right (reflow_greedily measure width segments) + streaming_greedy ⇒ Right (reflow_streaming_segments measure width segments) + balanced ⇒ Right (reflow_balanced measure width segments) + cubic ⇒ Right (reflow_for_objective cubic_raggedness measure width segments) + final_line_free ⇒ Right (reflow_for_objective free_final_line measure width segments) + minimax ⇒ Right (reflow_for_objective minimax_raggedness measure width segments) + threshold_balanced ⇒ Right (reflow_threshold_balanced measure width segments) + knuth_plass ⇒ + case reflow_knuth_plass (default_knuth_settings width) measure width segments of + Left reason ⇒ Left reason + Right plan ⇒ Right (plain_lines plan) diff --git a/src/IB/Reflow/Arithmetic.idric b/src/IB/Reflow/Arithmetic.idric new file mode 100644 index 00000000..6d150b94 --- /dev/null +++ b/src/IB/Reflow/Arithmetic.idric @@ -0,0 +1,31 @@ +module IB.Reflow.Arithmetic + +-- Number is the maintained nonnegative whole-number surface. Subtracting +-- directly via (-) requires a Neg instance the pinned compiler does not have +-- for Number. This saturating difference is total, explicit and portable. +-- It is a correctness baseline, not a target-specific fast subtraction. +public export +nonnegative_difference : Number → Number → Number +nonnegative_difference amount Z = amount +nonnegative_difference Z (S taken) = Z +nonnegative_difference (S amount) (S taken) = + nonnegative_difference amount taken + +-- Saturated division for TeX badness. The pinned Number/Nat surface lacks +-- Integral, so div is not available. Compute the largest representable +-- quotient <= 10000 using 14 descending binary weights; this is exact +-- until the TeX badness saturation limit. No repeated-subtraction loop. +quotient_bits : Number → Number → Number → List Number → Number +quotient_bits numerator denominator accumulated [] = accumulated +quotient_bits numerator denominator accumulated (weight :: rest) = + let proposal = accumulated + weight in + if proposal <= 10000 && proposal * denominator <= numerator + then quotient_bits numerator denominator proposal rest + else quotient_bits numerator denominator accumulated rest + +public export +capped_badness_quotient : Number → Number → Number +capped_badness_quotient numerator denominator = + if denominator ≟ 0 then 10000 + else quotient_bits numerator denominator 0 + [8192, 4096, 2048, 1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1] diff --git a/src/IB/Reflow/Balanced.idric b/src/IB/Reflow/Balanced.idric new file mode 100644 index 00000000..9cf3328b --- /dev/null +++ b/src/IB/Reflow/Balanced.idric @@ -0,0 +1,64 @@ +module IB.Reflow.Balanced + +import Data.Text +import IB.Reflow.Arithmetic + +-- Optimize the entire paragraph rather than deciding each break immediately. +-- Each suffix keeps its best known layout; candidate breaks are tested against +-- those suffixes. The quadratic number of candidate breaks and retained plans +-- cost more time/memory than greedy reflow, especially with Text concatenation. +-- The metric is total squared unused width, INCLUDING the final line. +-- Use the pinned compiler's executable ASCII arithmetic operators here; +-- Unicode minus/multiply remain an Idriç frontend compatibility gap. +-- Unbreakable overwide segments are kept intact and incur zero slack penalty; +-- their overrun must be dealt with by the rendering boundary, not clipped here. +record LinePlan where + constructor Planned + penalty : Number + lines : List Text + +unused_width : Number → Number → Number +unused_width available used = + if used <= available then nonnegative_difference available used else 0 + +candidate : (Text → Number) → Number → Text → LinePlan → LinePlan +candidate measure width line (Planned tail_penalty tail_lines) = + let slack = unused_width width (measure line) in + Planned (slack * slack + tail_penalty) (line :: tail_lines) + +prefer : LinePlan → LinePlan → LinePlan +prefer first second = + if first.penalty <= second.penalty then first else second + +-- The suffix plans align with the remaining input segments. An equal-cost +-- choice retains the earlier (shorter first-line) candidate deterministically. +consider_longer : (Text → Number) → Number → Text → List Text → + List LinePlan → LinePlan → LinePlan +consider_longer measure width current [] plans best = best +consider_longer measure width current words [] best = best +consider_longer measure width current (next :: rest) (tail :: tails) best = + let extended = current ++ " " ++ next in + -- Check every candidate even after a non-fitting prefix. + if measure extended <= width + then consider_longer measure width extended rest tails + (prefer best (candidate measure width extended tail)) + else consider_longer measure width extended rest tails best + +-- This is an executable suffix dynamic program, not an exponential search. +-- The first plan in the returned list is for the full supplied suffix. +suffix_plans : (Text → Number) → Number → List Text → List LinePlan +suffix_plans measure width [] = [Planned 0 []] +suffix_plans measure width (first :: rest) = + case suffix_plans measure width rest of + [] ⇒ [] + tail :: later ⇒ + let initial = candidate measure width first tail in + let best = consider_longer measure width first rest later initial in + best :: tail :: later + +public export +reflow_balanced : (Text → Number) → Number → List Text → List Text +reflow_balanced measure width segments = + case suffix_plans measure width segments of + [] ⇒ [] + (Planned _ lines) :: _ ⇒ lines diff --git a/src/IB/Reflow/Greedy.idric b/src/IB/Reflow/Greedy.idric new file mode 100644 index 00000000..75a886f7 --- /dev/null +++ b/src/IB/Reflow/Greedy.idric @@ -0,0 +1,21 @@ +module IB.Reflow.Greedy + +import Data.Text + +-- Single forward pass through the supplied segments. This can be adapted to +-- streaming input and retains only one unfinished line. A greedy line may +-- leave the following lines ragged, and width measurement may itself be costly. +-- Never discard or split a segment just to satisfy the line width. +fill_line : (Text → Number) → Number → Text → List Text → List Text +fill_line measure width current [] = [current] +fill_line measure width current (next :: rest) = + let extended = current ++ " " ++ next in + if measure extended <= width + then fill_line measure width extended rest + else current :: fill_line measure width next rest + +public export +reflow_greedily : (Text → Number) → Number → List Text → List Text +reflow_greedily measure width [] = [] +reflow_greedily measure width (first :: rest) = + fill_line measure width first rest diff --git a/src/IB/Reflow/Incremental.idric b/src/IB/Reflow/Incremental.idric new file mode 100644 index 00000000..a593f005 --- /dev/null +++ b/src/IB/Reflow/Incremental.idric @@ -0,0 +1,61 @@ +module IB.Reflow.Incremental + +import Data.Text +import IB.Reflow + +-- Incremental reflow is a cache strategy across revisions, not a competing +-- line-break objective. Reuse a paragraph only when its source segments, width, +-- chosen manner, and shaping/measurement identity all match. +-- An editor or view owns paragraph IDs and separate cache entries; this module +-- never becomes the canonical source and must not override source revisions. +public export +record ParagraphCache where + constructor CachedParagraph + cached_width : Number + cached_how : reflow_adverb + cached_metric_key : Text + cached_segments : List Text + cached_output : List Text + +public export +record CacheResult where + constructor ParagraphResult + resulting_cache : ParagraphCache + reused_previous : Bool + +same_manner : reflow_adverb → reflow_adverb → Bool +same_manner promptly promptly = True +same_manner streamingly streamingly = True +same_manner evenly evenly = True +same_manner minimizing_raggedness minimizing_raggedness = True +same_manner ignoring_final_line ignoring_final_line = True +same_manner minimizing_worst_gap minimizing_worst_gap = True +same_manner without_extra_lines without_extra_lines = True +same_manner typographically typographically = True +same_manner _ _ = False + +cache_matches : ReflowRequirements → Text → List Text → + ParagraphCache → Bool +cache_matches (Requirements width manner) metric_key segments + (CachedParagraph old_width old_manner old_metric old_segments _) = + width ≟ old_width && same_manner manner old_manner && + metric_key ≟ old_metric && segments ≟ old_segments + +public export +reflow_incrementally : ReflowRequirements → Text → (Text → Number) → + List Text → Maybe ParagraphCache → Either Text CacheResult +reflow_incrementally requirements@(Requirements width manner) + metric_key measure segments previous = + case previous of + Just cache ⇒ + if width > 0 && cache_matches requirements metric_key segments cache + then Right (ParagraphResult cache True) + else rebuild + Nothing ⇒ rebuild + where + rebuild : Either Text CacheResult + rebuild = + case reflow requirements measure segments of + Left problem ⇒ Left problem + Right lines ⇒ Right (ParagraphResult + (CachedParagraph width manner metric_key segments lines) False) diff --git a/src/IB/Reflow/KnuthPlass.idric b/src/IB/Reflow/KnuthPlass.idric new file mode 100644 index 00000000..cd9e8d99 --- /dev/null +++ b/src/IB/Reflow/KnuthPlass.idric @@ -0,0 +1,237 @@ +module IB.Reflow.KnuthPlass + +import Data.Text +import IB.Reflow.Arithmetic + +-- A real (deliberately restricted) Knuth–Plass box/glue line breaker: +-- ordered word boxes; break opportunities at interword spaces; optional +-- stretching/shrinking of those spaces; cubic adjustment badness; and an +-- adjacent-line fitness demerit. There is NO dictionary hyphenation or +-- discretionary penalty-node input yet. This is not TeX's complete engine. +-- +-- In contrast with quadratic raggedness, candidates carry the amount of +-- space adjustment required. Consumers wanting justification must use the +-- annotated line plan, not merely its plain-text projection. +public export +record KnuthSettings where + constructor KnuthOptions + stretch_per_gap : Number + shrink_per_gap : Number + emergency_stretch : Number + badness_tolerance : Number + fixed_line_penalty : Number + abrupt_fitness_penalty : Number + +public export +data SpaceAdjustment + = NoAdjustment + | SpreadSpaces Number Number + | CompressSpaces Number Number + +public export +data Fitness + = VeryTight + | Tight + | Neutral + | Loose + | VeryLoose + +public export +record JustifiedLine where + constructor Justified + line_text : Text + spacing : SpaceAdjustment + line_fitness : Fitness + +record KnuthPlan where + constructor PlannedKnuth + total_demerits : Number + justified_lines : List JustifiedLine + +-- Suffix plans are keyed by the fitness of the line immediately preceding +-- the suffix. This retains all five states needed for the transition cost; +-- keeping only one best suffix plan would give an incorrect optimum. +record PlansForSuffix where + constructor SuffixPlans + from_very_tight : Maybe KnuthPlan + from_tight : Maybe KnuthPlan + from_neutral : Maybe KnuthPlan + from_loose : Maybe KnuthPlan + from_very_loose : Maybe KnuthPlan + +next_plan : Fitness → PlansForSuffix → Maybe KnuthPlan +next_plan VeryTight plans = plans.from_very_tight +next_plan Tight plans = plans.from_tight +next_plan Neutral plans = plans.from_neutral +next_plan Loose plans = plans.from_loose +next_plan VeryLoose plans = plans.from_very_loose + +fitness_number : Fitness → Number +fitness_number VeryTight = 0 +fitness_number Tight = 1 +fitness_number Neutral = 2 +fitness_number Loose = 3 +fitness_number VeryLoose = 4 + +fitness_distance : Fitness → Fitness → Number +fitness_distance first later = + let a = fitness_number first in + let b = fitness_number later in + if a >= b then nonnegative_difference a b else nonnegative_difference b a + +is_last : List Text → Bool +is_last [] = True +is_last (_ :: _) = False + +-- TeX-style cubic badness with an integer numerator and denominator. The +-- exact ratio is also recorded, avoiding floating-point rounding in the plan. +adjustment_badness : Number → Number → Number +adjustment_badness amount capacity = + capped_badness_quotient + (100 * amount * amount * amount) + (capacity * capacity * capacity) + +within_tolerance : KnuthSettings → Number → Bool +within_tolerance settings badness = + badness <= settings.badness_tolerance + +spread_fitness : Number → Number → Fitness +spread_fitness amount capacity = + if amount ≟ 0 then Neutral + else if 2 * amount <= capacity then Neutral + else if amount <= capacity then Loose + else VeryLoose + +shrink_fitness : Number → Number → Fitness +shrink_fitness amount capacity = + if 2 * amount <= capacity then Tight else VeryTight + +-- A sole overwide word remains intact and is charged an explicit emergency +-- penalty. A multiword line cannot exceed the width unless its spaces have +-- enough declared shrink capacity. A final line is left unjustified. +shape_line : KnuthSettings → (Text → Number) → Number → + Text → Number → Bool → Maybe (JustifiedLine, Number) +shape_line settings measure width line gaps last = + let natural = measure line in + if natural > width && gaps ≟ 0 + then Just (Justified line NoAdjustment VeryTight, 10000) + else if last + then if natural <= width + then Just (Justified line NoAdjustment Neutral, 0) + else Nothing + else if natural <= width + then let unused = nonnegative_difference width natural in + if unused ≟ 0 + then Just (Justified line NoAdjustment Neutral, 0) + else let capacity = + gaps * settings.stretch_per_gap + + settings.emergency_stretch in + let badness = adjustment_badness unused capacity in + if capacity ≟ 0 || not (within_tolerance settings badness) + then Nothing + else Just + (Justified line (SpreadSpaces unused capacity) + (spread_fitness unused capacity), badness) + else let excess = nonnegative_difference natural width in + let capacity = gaps * settings.shrink_per_gap in + let badness = adjustment_badness excess capacity in + if capacity ≟ 0 || excess > capacity || + not (within_tolerance settings badness) + then Nothing + else Just + (Justified line (CompressSpaces excess capacity) + (shrink_fitness excess capacity), badness) + +consider : KnuthSettings → (Text → Number) → Number → Fitness → + Text → Number → Bool → PlansForSuffix → Maybe KnuthPlan +consider settings measure width previous line gap_count last tail_plans = + case shape_line settings measure width line gap_count last of + Nothing ⇒ Nothing + Just (geometry, badness) ⇒ + case next_plan geometry.line_fitness tail_plans of + Nothing ⇒ Nothing + Just (PlannedKnuth tail_cost tail_lines) ⇒ + let line_cost = settings.fixed_line_penalty + badness in + let fitness_jump = fitness_distance previous geometry.line_fitness in + let change_penalty = + if fitness_jump > 1 then settings.abrupt_fitness_penalty else 0 in + Just (PlannedKnuth + (line_cost * line_cost + change_penalty + tail_cost) + (geometry :: tail_lines)) + +prefer : Maybe KnuthPlan → Maybe KnuthPlan → Maybe KnuthPlan +prefer Nothing second = second +prefer first Nothing = first +prefer (Just first) (Just second) = + if first.total_demerits <= second.total_demerits + then Just first else Just second + +explore : KnuthSettings → (Text → Number) → Number → Fitness → + Text → Number → List Text → List PlansForSuffix → + Maybe KnuthPlan → Maybe KnuthPlan +explore settings measure width previous current gaps [] plans best = best +explore settings measure width previous current gaps words [] best = best +explore settings measure width previous current gaps + (word :: rest) (tail :: later) best = + let extended = current ++ " " ++ word in + let new_gaps = S gaps in + let proposal = consider settings measure width previous extended new_gaps (is_last rest) tail in + explore settings measure width previous extended new_gaps + rest later (prefer best proposal) + +best_from : KnuthSettings → (Text → Number) → Number → Fitness → + List Text → List PlansForSuffix → Maybe KnuthPlan +best_from settings measure width previous [] plans = Nothing +best_from settings measure width previous words [] = Nothing +best_from settings measure width previous (word :: rest) (tail :: later) = + let initial = consider settings measure width previous word Z (is_last rest) tail in + explore settings measure width previous word Z rest later initial + +terminal_plans : PlansForSuffix +terminal_plans = + let empty = Just (PlannedKnuth 0 []) in + SuffixPlans empty empty empty empty empty + +make_suffix_plan : KnuthSettings → (Text → Number) → Number → + List Text → List PlansForSuffix → PlansForSuffix +make_suffix_plan settings measure width words tails = + let a = best_from settings measure width VeryTight words tails in + let b = best_from settings measure width Tight words tails in + let c = best_from settings measure width Neutral words tails in + let d = best_from settings measure width Loose words tails in + let e = best_from settings measure width VeryLoose words tails in + SuffixPlans a b c d e + +all_suffix_plans : KnuthSettings → (Text → Number) → Number → + List Text → List PlansForSuffix +all_suffix_plans settings measure width [] = [terminal_plans] +all_suffix_plans settings measure width words@(_ :: rest) = + let tails = all_suffix_plans settings measure width rest in + make_suffix_plan settings measure width words tails :: tails + +public export +reflow_knuth_plass : KnuthSettings → (Text → Number) → Number → + List Text → Either Text (List JustifiedLine) +reflow_knuth_plass settings measure width [] = Right [] +reflow_knuth_plass settings measure width words = + case all_suffix_plans settings measure width words of + [] ⇒ Left "Knuth-Plass could not construct suffix plans" + plans :: _ ⇒ + case next_plan Neutral plans of + Nothing ⇒ Left "No admissible Knuth-Plass line-break plan" + Just (PlannedKnuth _ lines) ⇒ Right lines + +-- The defaults are a *policy*, not a benchmark finding: temporary stretch +-- scales with the viewport so readable ragged text is still feasible. +-- Shrink is disabled for the plain-text IB projection. A future native +-- renderer that actually consumes SpaceAdjustment can enable it explicitly. +public export +default_knuth_settings : Number → KnuthSettings +default_knuth_settings width = + KnuthOptions 2 0 width 10000 10 2000 + +public export +plain_lines : List JustifiedLine → List Text +plain_lines [] = [] +plain_lines ((Justified text adjustment fitness) :: rest) = + text :: plain_lines rest diff --git a/src/IB/Reflow/Objectives.idric b/src/IB/Reflow/Objectives.idric new file mode 100644 index 00000000..a8e5a4c0 --- /dev/null +++ b/src/IB/Reflow/Objectives.idric @@ -0,0 +1,80 @@ +module IB.Reflow.Objectives + +import Data.Text +import IB.Reflow.Arithmetic + +-- These three ways share the same suffix dynamic-programming engine, but ask +-- different mathematical questions. They cannot be interchanged blindly: +-- choosing a different objective may change observable line breaks. +public export +choice line_objective one_of + cubic_raggedness + free_final_line + minimax_raggedness + +record ObjectivePlan where + constructor PlannedObjective + objective_score : Number + objective_lines : List Text + +slack : Number → Number → Number +slack width used = + if used <= width then nonnegative_difference width used else 0 + +last_line : List Text → Bool +last_line [] = True +last_line (_ :: _) = False + +line_penalty : line_objective → Number → Bool → Number +line_penalty cubic_raggedness unused is_last = unused * unused * unused +line_penalty free_final_line unused True = 0 +line_penalty free_final_line unused False = unused * unused +line_penalty minimax_raggedness unused is_last = unused + +combine_score : line_objective → Number → Number → Number +combine_score minimax_raggedness first later = + if first >= later then first else later +combine_score _ first later = first + later + +candidate : line_objective → (Text → Number) → Number → + Text → ObjectivePlan → ObjectivePlan +candidate objective measure width line (PlannedObjective previous following) = + let unused = slack width (measure line) in + let penalty = line_penalty objective unused (last_line following) in + PlannedObjective (combine_score objective penalty previous) (line :: following) + +prefer : ObjectivePlan → ObjectivePlan → ObjectivePlan +prefer left right = + if left.objective_score <= right.objective_score then left else right + +-- Do not prune later breakpoints after one line fails to fit: an arbitrary +-- supplied shaped-text width metric is not guaranteed to be monotonic. +check_longer : line_objective → (Text → Number) → Number → Text → + List Text → List ObjectivePlan → ObjectivePlan → ObjectivePlan +check_longer objective measure width current [] plans best = best +check_longer objective measure width current values [] best = best +check_longer objective measure width current (word :: rest) (tail :: tails) best = + let extended = current ++ " " ++ word in + if measure extended <= width + then check_longer objective measure width extended rest tails + (prefer best (candidate objective measure width extended tail)) + else check_longer objective measure width extended rest tails best + +suffix_plans : line_objective → (Text → Number) → Number → + List Text → List ObjectivePlan +suffix_plans objective measure width [] = [PlannedObjective 0 []] +suffix_plans objective measure width (word :: rest) = + case suffix_plans objective measure width rest of + [] ⇒ [] + tail :: later ⇒ + let initial = candidate objective measure width word tail in + let best = check_longer objective measure width word rest later initial in + best :: tail :: later + +public export +reflow_for_objective : line_objective → (Text → Number) → Number → + List Text → List Text +reflow_for_objective objective measure width words = + case suffix_plans objective measure width words of + [] ⇒ [] + (PlannedObjective _ lines) :: _ ⇒ lines diff --git a/src/IB/Reflow/Source.idric b/src/IB/Reflow/Source.idric new file mode 100644 index 00000000..32a45081 --- /dev/null +++ b/src/IB/Reflow/Source.idric @@ -0,0 +1,221 @@ +module IB.Reflow.Source + +import Data.Text +import IB.Reflow +import IB.Reflow.Arithmetic +import IB.Reflow.SourceBreaks + +-- Unlike the earlier atomic-word reflow, this preserves source character +-- offsets and never normalizes or edits canonical source. The visible text is +-- a derived projection: paragraph controls are hidden, trailing display +-- spaces are suppressed, and a chosen soft-hyphen cut adds a visible "-". +-- This simple hyphen glyph is a policy, NOT locale-specific shaping. +hidden_character : Char → Bool +hidden_character char = + let point = ord char in + point ≟ 173 || point ≟ 8203 || + point ≟ 10 || point ≟ 13 || + point ≟ 133 || point ≟ 8232 || point ≟ 8233 + +without_controls : List Char → List Char +without_controls [] = [] +without_controls (char :: rest) = + if hidden_character char + then without_controls rest + else char :: without_controls rest + +strip_trailing_space : List Char → List Char +strip_trailing_space [] = [] +strip_trailing_space (' ' :: rest) = strip_trailing_space rest +strip_trailing_space (char :: rest) = char :: rest + +display_break_mark : BreakKind → Text → Text +display_break_mark DiscretionaryHyphen painted = painted ++ "-" +display_break_mark _ painted = painted + +source_line : Text → Number → SourceBreak → SourceLine +source_line source start (At finish kind) = + let span = FromSource start finish "" kind in + let raw = unpack (source_for_line source span) in + let painted = pack (reverse (strip_trailing_space (reverse (without_controls raw)))) in + FromSource start finish (display_break_mark kind painted) kind + +line_fits : (Text → Number) → Number → SourceLine → Bool +line_fits measure width line = + measure line.visible_text <= width + +terminal : BreakKind → Bool +terminal MandatoryBreak = True +terminal SourceEnd = True +terminal _ = False + +-- A greedy line chooses the farthest candidate that fits, stopping at the +-- first mandatory boundary. If not even the earliest candidate fits, emit +-- that atomic source run intact and report the overrun through line geometry. +-- It never invents an illegal break to stay inside the width. +last_fitting : Text → (Text → Number) → Number → Number → + SourceBreak → List SourceBreak → Maybe SourceBreak → SourceBreak +last_fitting source measure width start first [] chosen = + case chosen of + Nothing ⇒ first + Just answer ⇒ answer +last_fitting source measure width start first (candidate :: more) chosen = + let fits = line_fits measure width (source_line source start candidate) in + let best : Maybe SourceBreak = if fits then Just candidate else chosen in + if terminal candidate.break_kind + then case best of + Nothing ⇒ first + Just answer ⇒ answer + else last_fitting source measure width start first more best + +select_greedy : Text → (Text → Number) → Number → Number → + List SourceBreak → SourceBreak +select_greedy source measure width start [] = At start SourceEnd +select_greedy source measure width start (first :: later) = + if terminal first.break_kind + then first + else let first_fit = line_fits measure width + (source_line source start first) in + last_fitting source measure width start first later + (if first_fit then Just first else Nothing) + +after_cut : Number → List SourceBreak → List SourceBreak +after_cut offset [] = [] +after_cut offset (cut :: rest) = + if cut.source_offset <= offset then after_cut offset rest + else cut :: rest + +-- A valid selection consumes at least one candidate. Bound recursive descent +-- by the original candidate count; an exhausted budget refuses the projection +-- rather than silently dropping source or asserting an unchecked termination. +greedy_lines : Number → Text → (Text → Number) → Number → Number → + List SourceBreak → Either Text (List SourceLine) +greedy_lines remaining_choices source measure width start [] = Right [] +greedy_lines Z source measure width start cuts = + Left "Reflow candidate budget exhausted" +greedy_lines (S remaining_choices) source measure width start cuts = + let chosen = select_greedy source measure width start cuts in + case greedy_lines remaining_choices source measure width chosen.source_offset + (after_cut chosen.source_offset cuts) of + Left message ⇒ Left message + Right later_lines ⇒ Right (source_line source start chosen :: later_lines) + +record SourcePlan where + constructor PlannedSource + total_cost : Number + number_of_lines : Number + lines : List SourceLine + +badness : reflow_adverb → Number → Number → Bool → Number +badness mode width used last = + let unused = nonnegative_difference width used in + if used > width + then let excess = nonnegative_difference used width in + 1000000000 + excess * excess + else case mode of + ignoring_final_line ⇒ if last then 0 else unused * unused + minimizing_raggedness ⇒ unused * unused * unused + _ ⇒ unused * unused + +combined_cost : reflow_adverb → Number → Number → Number +combined_cost minimizing_worst_gap first later = + if first >= later then first else later +combined_cost _ first later = first + later + +soft_hyphen_cost : BreakKind → Number +soft_hyphen_cost DiscretionaryHyphen = 50 +soft_hyphen_cost _ = 0 + +candidate_plan : Text → reflow_adverb → (Text → Number) → Number → + Number → SourceBreak → SourcePlan → SourcePlan +candidate_plan source mode measure width start cut (PlannedSource tail_cost count tail) = + let line = source_line source start cut in + let last = cut.source_offset ≟ length (unpack source) in + let cost = badness mode width (measure line.visible_text) last in + let adjusted = cost + soft_hyphen_cost cut.break_kind in + PlannedSource (combined_cost mode adjusted tail_cost) + (S count) (line :: tail) + +prefer_plan : reflow_adverb → SourcePlan → SourcePlan → SourcePlan +prefer_plan without_extra_lines first second = + if first.number_of_lines < second.number_of_lines then first + else if first.number_of_lines > second.number_of_lines then second + else if first.total_cost <= second.total_cost then first else second +prefer_plan mode first second = + if first.total_cost <= second.total_cost then first else second + +update_best : reflow_adverb → Maybe SourcePlan → SourcePlan → Maybe SourcePlan +update_best mode Nothing candidate = Just candidate +update_best mode (Just current) candidate = + Just (prefer_plan mode current candidate) + +-- Candidate plans are aligned with the source offsets of cuts. The first +-- candidate is always admissible, even if it exceeds the width; later ones +-- must fit. Required boundaries cap the search, so a plan cannot cross them. +consider_cuts : Text → reflow_adverb → (Text → Number) → Number → + Number → Bool → List SourceBreak → List SourcePlan → + Maybe SourcePlan → Maybe SourcePlan +consider_cuts source mode measure width start first [] plans best = best +consider_cuts source mode measure width start first cuts [] best = best +consider_cuts source mode measure width start first + (cut :: rest) (tail :: tails) best = + let line = source_line source start cut in + let admissible = first || line_fits measure width line in + if terminal cut.break_kind + then if admissible + then update_best mode best (candidate_plan source mode measure width start cut tail) + else best + else if admissible + then consider_cuts source mode measure width start False rest tails + (update_best mode best (candidate_plan source mode measure width start cut tail)) + else consider_cuts source mode measure width start False rest tails best + +plan_from : Text → reflow_adverb → (Text → Number) → Number → + Number → List SourceBreak → List SourcePlan → SourcePlan +plan_from source mode measure width start cuts tails = + case consider_cuts source mode measure width start True cuts tails Nothing of + Just answer ⇒ answer + Nothing ⇒ PlannedSource 0 0 [] + +suffix_plans : Text → reflow_adverb → (Text → Number) → Number → + List SourceBreak → List SourcePlan +suffix_plans source mode measure width [] = [PlannedSource 0 0 []] +suffix_plans source mode measure width (At offset kind :: later) = + let tails = suffix_plans source mode measure width later in + let plan = plan_from source mode measure width offset later tails in + plan :: tails + +optimal_lines : Text → reflow_adverb → (Text → Number) → Number → + List SourceBreak → List SourceLine +optimal_lines source mode measure width cuts = + let tails = suffix_plans source mode measure width cuts in + let (PlannedSource _ _ lines) = plan_from source mode measure width 0 cuts tails in + lines + +-- On the source-bound representation the following requests are genuinely +-- implemented: greedy, minimum squared/cubic/maximum slack, final-line-free +-- and *exact* minimum-line-count/secondarily minimum-square-slack. +-- The last differs from the atomic-word threshold heuristic, by design. +-- Streaming requires a lifetime-aware source-offset accumulator, and TeX +-- needs actual variable-width glue/penalty nodes, so those fail explicitly. +public export +reflow_at_breaks : ReflowRequirements → (Text → Number) → Text → + List SourceBreak → Either Text (List SourceLine) +reflow_at_breaks (Requirements Z _) measure source cuts = + Left "Reflow width must be positive" +reflow_at_breaks (Requirements width mode) measure source cuts = + if not (valid_source_breaks source cuts) + then Left "Invalid, unsafe or incomplete source break opportunities" + else case unpack source of + [] ⇒ Right [] + _ ⇒ case mode of + promptly ⇒ greedy_lines (length cuts) source measure width 0 cuts + streamingly ⇒ Left "Source-offset streaming has not been qualified" + typographically ⇒ Left "Source-backed adjustable glue is not qualified" + _ ⇒ Right (optimal_lines source mode measure width cuts) + +public export +reflow_source : ReflowRequirements → (Text → Number) → Text → + Either Text (List SourceLine) +reflow_source requirements measure source = + reflow_at_breaks requirements measure source (source_breaks source) diff --git a/src/IB/Reflow/SourceBreaks.idric b/src/IB/Reflow/SourceBreaks.idric new file mode 100644 index 00000000..f7ba27cf --- /dev/null +++ b/src/IB/Reflow/SourceBreaks.idric @@ -0,0 +1,201 @@ +module IB.Reflow.SourceBreaks + +import Data.Text +import IB.Reflow.Arithmetic + +-- A source-bound list of possible line breaks. Offsets count decoded Unicode +-- code points (elements of unpack), NOT UTF-8 bytes, UTF-16 units or glyphs. +-- A break is the position immediately AFTER source_offset characters. +-- The source text itself remains canonical. No renderer output is reparsed. +public export +data BreakKind + = PossibleBreak + | MandatoryBreak + | DiscretionaryHyphen + | ZeroWidthBreak + | SourceEnd + +public export +record SourceBreak where + constructor At + source_offset : Number + break_kind : BreakKind + +public export +record SourceLine where + constructor FromSource + first_codepoint : Number + after_last_codepoint : Number + visible_text : Text + ending : BreakKind + +is_han : Char → Bool +is_han value = + let number = ord value in + (number >= 19968 && number <= 40959) || + (number >= 13312 && number <= 19903) + +-- Deliberately bounded Unicode tailoring, NOT the complete UAX #14. +-- ASCII SPACE, ZERO WIDTH SPACE, SOFT HYPHEN, hard line endings and selected +-- Han–Han boundaries produce candidates. NBSP/WORD JOINER and emoji ZWJ do +-- not generate breaks. Complete Unicode/locale break data can supply an +-- independently qualified list of SourceBreak values to the same solver. +hard_line_end : Char → List Char → Bool +hard_line_end '\r' ('\n' :: _) = False +hard_line_end '\r' _ = True +hard_line_end '\n' _ = True +hard_line_end value _ = + let number = ord value in + number ≟ 133 || number ≟ 8232 || number ≟ 8233 + +classify : Char → List Char → Maybe BreakKind +classify value rest = + if hard_line_end value rest then Just MandatoryBreak + else if ord value ≟ 173 then Just DiscretionaryHyphen + else if ord value ≟ 8203 then Just ZeroWidthBreak + else if value ≟ ' ' then + case rest of + next :: _ ⇒ + if hard_line_end next [] || ord next ≟ 8205 || ord next ≟ 8288 + then Nothing else Just PossibleBreak + [] ⇒ Nothing + else if is_han value then + case rest of + next :: _ ⇒ if is_han next then Just PossibleBreak else Nothing + [] ⇒ Nothing + else Nothing + +is_empty : List Char → Bool +is_empty [] = True +is_empty (_ :: _) = False + +scan_breaks : Number → List Char → List SourceBreak +scan_breaks offset [] = [At offset SourceEnd] +scan_breaks offset (value :: rest) = + let later = scan_breaks (S offset) rest in + case classify value rest of + Nothing ⇒ later + Just kind ⇒ + if is_empty rest + then case kind of + MandatoryBreak ⇒ [At (S offset) MandatoryBreak] + _ ⇒ later + else At (S offset) kind :: later + +public export +source_breaks : Text → List SourceBreak +source_breaks text = scan_breaks 0 (unpack text) + +-- Checks do not establish all of UAX #14/#29. They *do* fail closed on +-- malformed ranges, duplicate/reordered cuts, missing mandatory breaks, cuts +-- inside CRLF or after ZWJ/word joiner, and cuts before common combining marks. +-- Full segmentation is a separate, versioned provider qualification. +at_position : Number → List Char → Maybe Char +at_position Z [] = Nothing +at_position Z (value :: _) = Just value +at_position (S position) [] = Nothing +at_position (S position) (_ :: rest) = at_position position rest + +before : Number → List Char → Maybe Char +before Z chars = Nothing +before (S previous) chars = at_position previous chars + +is_combining_mark : Char → Bool +is_combining_mark value = + let number = ord value in + (number >= 768 && number <= 879) || + (number >= 6832 && number <= 6911) || + (number >= 7616 && number <= 7679) || + (number >= 8400 && number <= 8447) || + (number >= 65056 && number <= 65071) + +safe_cut : List Char → Number → BreakKind → Bool +safe_cut chars offset kind = + case (before offset chars, at_position offset chars) of + (Just previous, Just next) ⇒ + not (previous ≟ '\r' && next ≟ '\n') && + not (ord previous ≟ 8205 || ord previous ≟ 8288) && + not (ord next ≟ 8205 || ord next ≟ 8288) && + (case kind of + MandatoryBreak ⇒ True + _ ⇒ not (is_combining_mark next)) && + (case kind of + MandatoryBreak ⇒ True + _ ⇒ not (hard_line_end next [])) + _ ⇒ True + +requires_break : List Char → Number → Bool +requires_break chars offset = + case before offset chars of + Nothing ⇒ False + Just previous ⇒ + let remaining = drop offset chars in + hard_line_end previous remaining + +same_kind : BreakKind → BreakKind → Bool +same_kind PossibleBreak PossibleBreak = True +same_kind MandatoryBreak MandatoryBreak = True +same_kind DiscretionaryHyphen DiscretionaryHyphen = True +same_kind ZeroWidthBreak ZeroWidthBreak = True +same_kind SourceEnd SourceEnd = True +same_kind _ _ = False + +-- For a declared mandatory cut, check that the source actually forces it. +-- The end marker is allowed only at the exact end, except that a hard final +-- newline is represented by MandatoryBreak instead. +kind_is_sound : List Char → Number → BreakKind → Bool +kind_is_sound chars offset MandatoryBreak = requires_break chars offset +kind_is_sound chars offset DiscretionaryHyphen = + case before offset chars of + Just previous ⇒ ord previous ≟ 173 + Nothing ⇒ False +kind_is_sound chars offset ZeroWidthBreak = + case before offset chars of + Just previous ⇒ ord previous ≟ 8203 + Nothing ⇒ False +kind_is_sound chars offset PossibleBreak = + not (requires_break chars offset) +kind_is_sound chars offset SourceEnd = + offset ≟ length chars && not (requires_break chars offset) + +well_ordered : List Char → Number → List SourceBreak → Bool +well_ordered chars previous [] = previous ≟ length chars +well_ordered chars previous (At offset kind :: rest) = + offset > previous && offset <= length chars && + safe_cut chars offset kind && + kind_is_sound chars offset kind && + well_ordered chars offset rest + +has_mandatory : Number → List SourceBreak → Bool +has_mandatory offset [] = False +has_mandatory offset (At position kind :: rest) = + if offset ≟ position + then same_kind kind MandatoryBreak + else has_mandatory offset rest + +all_hard_breaks : Number → List Char → List SourceBreak → Bool +all_hard_breaks offset [] breaks = True +all_hard_breaks offset (value :: rest) breaks = + let next = S offset in + if hard_line_end value rest + then has_mandatory next breaks && all_hard_breaks next rest breaks + else all_hard_breaks next rest breaks + +public export +valid_source_breaks : Text → List SourceBreak → Bool +valid_source_breaks text breaks = + let chars = unpack text in + case chars of + [] ⇒ case breaks of + [] ⇒ True + [At Z SourceEnd] ⇒ True + _ ⇒ False + _ ⇒ well_ordered chars 0 breaks && all_hard_breaks 0 chars breaks + +-- Reconstruct EXACT decoded source from offsets, regardless of paint spelling. +-- This is an independent identity oracle for layouts with discretionary marks. +public export +source_for_line : Text → SourceLine → Text +source_for_line source line = + pack (take (nonnegative_difference line.after_last_codepoint line.first_codepoint) + (drop line.first_codepoint (unpack source))) diff --git a/src/IB/Reflow/Streaming.idric b/src/IB/Reflow/Streaming.idric new file mode 100644 index 00000000..0fab9185 --- /dev/null +++ b/src/IB/Reflow/Streaming.idric @@ -0,0 +1,101 @@ +module IB.Reflow.Streaming + +import Data.Text + +-- A real push interface: each chunk may end inside a word. The caller owns +-- paragraph boundaries, and can emit completed lines before the source ends. +-- Only the current line and one incomplete word are retained; an extremely +-- long unbreakable word is necessarily retained until its next boundary. +-- The list is reversed so receiving one character takes constant list work. +public export +record ReflowStream where + constructor Streaming + stream_width : Number + reversed_unfinished_word : List Char + unfinished_line : Maybe Text + +public export +record StreamStep where + constructor Emitted + next_stream : ReflowStream + completed_lines : List Text + +public export +begin_stream : Number → Either Text ReflowStream +begin_stream Z = Left "Reflow width must be positive" +begin_stream width = Right (Streaming width [] Nothing) + +-- We deliberately normalize whitespace to one display space. The source +-- bytes and semantic spans remain owned by IB.Information, not this renderer. +separator : Char → Bool +separator char = + char ≟ ' ' || char ≟ '\n' || char ≟ '\r' || char ≟ '\t' + +-- A finished word may complete the previous line, which can be published +-- immediately. Unlike the batch function, no future word is needed afterward. +put_word : (Text → Number) → Number → Maybe Text → Text → + (Maybe Text, List Text) +put_word measure width Nothing word = (Just word, []) +put_word measure width (Just line) word = + let extended = line ++ " " ++ word in + if measure extended <= width + then (Just extended, []) + else (Just word, [line]) + +complete_word : (Text → Number) → ReflowStream → + (ReflowStream, List Text) +complete_word measure state@(Streaming width [] line) = (state, []) +complete_word measure (Streaming width reversed_word line) = + let word = pack (reverse reversed_word) in + let (next_line, emitted) = put_word measure width line word in + (Streaming width [] next_line, emitted) + +scan_chunk : (Text → Number) → ReflowStream → List Char → List Text → + StreamStep +scan_chunk measure state [] emitted_reversed = + Emitted state (reverse emitted_reversed) +scan_chunk measure state@(Streaming width word line) (char :: rest) emitted_reversed = + if separator char + then let (next, emitted) = complete_word measure state in + scan_chunk measure next rest (reverse emitted ++ emitted_reversed) + else scan_chunk measure (Streaming width (char :: word) line) + rest emitted_reversed + +-- feed_chunk accepts arbitrarily split source chunks. No pending word is +-- published prematurely at the end of a chunk. +public export +feed_chunk : (Text → Number) → ReflowStream → Text → StreamStep +feed_chunk measure state chunk = + scan_chunk measure state (unpack chunk) [] + +-- Explicit paragraph or stream completion flushes the last line. +public export +finish_stream : (Text → Number) → ReflowStream → List Text +finish_stream measure state = + let (Streaming _ _ line, emitted) = complete_word measure state in + case line of + Nothing ⇒ emitted + Just value ⇒ emitted ++ [value] + +-- Paragraph boundaries are semantic input, not inferred from arbitrary +-- transport chunking. The returned state can receive the next paragraph. +public export +end_paragraph : (Text → Number) → ReflowStream → + (ReflowStream, List Text) +end_paragraph measure state@(Streaming width _ _) = + (Streaming width [] Nothing, finish_stream measure state) + +-- Batch adapter over the same one-word-at-a-time state transition. It does +-- not materialize a joined paragraph or change token boundaries. +-- Calling feed_chunk instead is the genuine streaming source interface. +run_segments : (Text → Number) → Number → Maybe Text → List Text → List Text +run_segments measure width Nothing [] = [] +run_segments measure width (Just line) [] = [line] +run_segments measure width line (word :: rest) = + let (next, emitted) = put_word measure width line word in + emitted ++ run_segments measure width next rest + +public export +reflow_streaming_segments : (Text → Number) → Number → List Text → List Text +reflow_streaming_segments measure width segments = + run_segments measure width Nothing segments diff --git a/src/IB/Reflow/Threshold.idric b/src/IB/Reflow/Threshold.idric new file mode 100644 index 00000000..92ff9f60 --- /dev/null +++ b/src/IB/Reflow/Threshold.idric @@ -0,0 +1,27 @@ +module IB.Reflow.Threshold + +import Data.Text +import IB.Reflow.Greedy + +-- A CSS text-wrap:balance-style approximation (not a browser's CSS engine). +-- First determine the minimum greedy line count at the available width. +-- Then repeatedly narrow the effective width while that count stays fixed. +-- This often makes lines less uneven than greedy without quadratic DP state. +-- Cost is repeated scans, up to available_width scans, and quality is only +-- heuristic: the effective-width choice need not minimize raggedness. +tighten : (Text → Number) → Number → Number → List Text → + List Text → List Text +tighten measure Z wanted words best = best +tighten measure (S Z) wanted words best = best +tighten measure (S (S narrower)) wanted words best = + let candidate = reflow_greedily measure (S narrower) words in + if length candidate <= wanted + then tighten measure (S narrower) wanted words candidate + else best + +public export +reflow_threshold_balanced : (Text → Number) → Number → + List Text → List Text +reflow_threshold_balanced measure width words = + let first = reflow_greedily measure width words in + tighten measure width (length first) words first diff --git a/src/ReflowSmoke.idric b/src/ReflowSmoke.idric new file mode 100644 index 00000000..e4f8431a --- /dev/null +++ b/src/ReflowSmoke.idric @@ -0,0 +1,187 @@ +module ReflowSmoke + +import Data.Text +import IB.Reflow +import IB.Reflow.Streaming +import IB.Reflow.KnuthPlass +import IB.Reflow.Incremental +import IB.Reflow.Arithmetic +import System + +character_width : Text → Number +character_width value = length (unpack value) + +matches : List Text → Either Text (List Text) → Bool +matches wanted (Right actual) = wanted ≟ actual +matches wanted _ = False + +join_lines : List Text → Text +join_lines [] = "" +join_lines [line] = line +join_lines (line :: rest) = line ++ " " ++ join_lines rest + +keeps_words : List Text → Either Text (List Text) → Bool +keeps_words original (Right wrapped) = + join_lines original ≟ join_lines wrapped +keeps_words original _ = False + +has_error : Either Text (List Text) → Bool +has_error (Left _) = True +has_error _ = False + +-- Transport chunks are not tokens. In particular "eee" + "eee" must +-- remain one six-character word. We also assert early line emission. +streaming_case : (Bool, Bool) +streaming_case = + case begin_stream 8 of + Left _ ⇒ (False, False) + Right start ⇒ + let Emitted next1 published1 = feed_chunk character_width start "aa bb cc dd " in + let Emitted next2 published2 = feed_chunk character_width next1 "eee" in + let Emitted next3 published3 = feed_chunk character_width next2 "eee" in + let all_lines = + published1 ++ published2 ++ published3 ++ finish_stream character_width next3 in + (published1 ≟ ["aa bb cc"], all_lines ≟ ["aa bb cc", "dd", "eeeeee"]) + +paragraph_case : Bool +paragraph_case = + case begin_stream 8 of + Left _ ⇒ False + Right initial ⇒ + let Emitted before _ = feed_chunk character_width initial "aa bb" in + let (after, first_paragraph) = end_paragraph character_width before in + let Emitted last _ = feed_chunk character_width after "cc dd" in + first_paragraph ≟ ["aa bb"] && finish_stream character_width last ≟ ["cc dd"] + +preserves_in_every_mode : List reflow_adverb → List Text → Bool +preserves_in_every_mode [] words = True +preserves_in_every_mode (mode :: more) words = + keeps_words words (reflow (Requirements 10 mode) character_width words) && + preserves_in_every_mode more words + +overwide_in_every_mode : List reflow_adverb → Bool +overwide_in_every_mode [] = True +overwide_in_every_mode (mode :: more) = + matches ["abcdefghij", "x"] + (reflow (Requirements 8 mode) character_width ["abcdefghij", "x"]) && + overwide_in_every_mode more + +empty_in_every_mode : List reflow_adverb → Bool +empty_in_every_mode [] = True +empty_in_every_mode (mode :: more) = + matches [] (reflow (Requirements 8 mode) character_width []) && + empty_in_every_mode more + +zero_refused_in_every_mode : List reflow_adverb → Bool +zero_refused_in_every_mode [] = True +zero_refused_in_every_mode (mode :: more) = + has_error (reflow (Requirements 0 mode) character_width ["x"]) && + zero_refused_in_every_mode more + +has_adjustment : List JustifiedLine → Bool +has_adjustment [] = False +has_adjustment ((Justified _ NoAdjustment _) :: rest) = has_adjustment rest +has_adjustment (_ :: _) = True + +knuth_plan_case : Bool +knuth_plan_case = + case reflow_knuth_plass (default_knuth_settings 8) character_width 8 ["aa", "bb", "cc", "dd", "eeeeee"] of + Left _ ⇒ False + Right lines ⇒ + has_adjustment lines && + join_lines (plain_lines lines) ≟ "aa bb cc dd eeeeee" + +incremental_case : Bool +incremental_case = + let request = Requirements 8 evenly in + let words = ["aa", "bb", "cc", "dd", "eeeeee"] in + case reflow_incrementally request "font-1" character_width words Nothing of + Left _ ⇒ False + Right built ⇒ + case reflow_incrementally request "font-1" character_width words (Just built.resulting_cache) of + Left _ ⇒ False + Right reused ⇒ + case reflow_incrementally request "font-1" character_width ["aa", "bb", "changed"] (Just built.resulting_cache) of + Left _ ⇒ False + Right edited ⇒ + case reflow_incrementally (Requirements 7 evenly) "font-1" character_width words (Just built.resulting_cache) of + Left _ ⇒ False + Right resized ⇒ + case reflow_incrementally request "font-2" character_width words (Just built.resulting_cache) of + Left _ ⇒ False + Right new_metric ⇒ + not built.reused_previous && + reused.reused_previous && + not edited.reused_previous && + not resized.reused_previous && + not new_metric.reused_previous + +begin_zero_refused : Bool +begin_zero_refused = + case begin_stream 0 of + Left _ ⇒ True + Right _ ⇒ False + +main : IO () +main = do + let words = ["aa", "bb", "cc", "dd", "eeeeee"] + let greedy_lines = ["aa bb cc", "dd", "eeeeee"] + let evenly_lines = ["aa bb", "cc dd", "eeeeee"] + let greedy_ok = matches greedy_lines (reflow (Requirements 8 promptly) character_width words) + let streaming_ok = matches greedy_lines (reflow (Requirements 8 streamingly) character_width words) + let balanced_ok = matches evenly_lines (reflow (Requirements 8 evenly) character_width words) + let threshold_ok = matches evenly_lines (reflow (Requirements 8 without_extra_lines) character_width words) + + -- This paragraph discriminates different dynamic-programming objectives. + let odd = ["xxxx", "xxxxx", "xx", "xx", "xxxxxx", "x", "xx"] + let cubic_ok = matches ["xxxx xxxxx", "xx xx", "xxxxxx", "x xx"] (reflow (Requirements 10 minimizing_raggedness) character_width odd) + let final_free_ok = matches ["xxxx xxxxx", "xx xx", "xxxxxx x", "xx"] (reflow (Requirements 10 ignoring_final_line) character_width odd) + let minimax_ok = matches ["xxxx", "xxxxx", "xx xx", "xxxxxx", "x xx"] (reflow (Requirements 10 minimizing_worst_gap) character_width odd) + let squared_ok = matches ["xxxx", "xxxxx xx", "xx xxxxxx", "x xx"] (reflow (Requirements 10 evenly) character_width odd) + + let modes = [promptly, streamingly, evenly, minimizing_raggedness, + ignoring_final_line, minimizing_worst_gap, without_extra_lines, + typographically] + let content_preserved = preserves_in_every_mode modes odd + let long_word_intact = overwide_in_every_mode modes + let empty_intact = empty_in_every_mode modes + let zero_refused = zero_refused_in_every_mode modes + let (emits_early, chunk_invariant) = streaming_case + let paragraphs_ok = paragraph_case + let arithmetic_ok = + capped_badness_quotient 18 3 ≟ 6 && + capped_badness_quotient 11000 1 ≟ 10000 && + capped_badness_quotient 0 100 ≟ 0 && + nonnegative_difference 10 3 ≟ 7 && + nonnegative_difference 3 10 ≟ 0 + let knuth_ok = knuth_plan_case + let incremental_ok = incremental_case + let zero_stream_refused = begin_zero_refused + + putStrLn ("reflow-greedy=" ++ show greedy_ok) + putStrLn ("reflow-streaming-batch=" ++ show streaming_ok) + putStrLn ("reflow-streaming-early=" ++ show emits_early) + putStrLn ("reflow-streaming-chunks=" ++ show chunk_invariant) + putStrLn ("reflow-streaming-paragraphs=" ++ show paragraphs_ok) + putStrLn ("reflow-balanced=" ++ show balanced_ok) + putStrLn ("reflow-cubic=" ++ show cubic_ok) + putStrLn ("reflow-final-line-free=" ++ show final_free_ok) + putStrLn ("reflow-minimax=" ++ show minimax_ok) + putStrLn ("reflow-threshold=" ++ show threshold_ok) + putStrLn ("reflow-squared-discriminates=" ++ show squared_ok) + putStrLn ("reflow-preserves-content=" ++ show content_preserved) + putStrLn ("reflow-overwide-intact=" ++ show long_word_intact) + putStrLn ("reflow-empty=" ++ show empty_intact) + putStrLn ("reflow-zero-refused=" ++ show zero_refused) + putStrLn ("reflow-stream-zero-refused=" ++ show zero_stream_refused) + putStrLn ("reflow-knuth-spacings=" ++ show knuth_ok) + putStrLn ("reflow-incremental-revisions=" ++ show incremental_ok) + putStrLn ("reflow-arithmetic=" ++ show arithmetic_ok) + + if greedy_ok && streaming_ok && emits_early && chunk_invariant && + paragraphs_ok && balanced_ok && cubic_ok && final_free_ok && + minimax_ok && threshold_ok && squared_ok && content_preserved && + long_word_intact && empty_intact && zero_refused && zero_stream_refused && + knuth_ok && incremental_ok && arithmetic_ok + then pure () + else exitFailure diff --git a/src/SourceReflowSmoke.idric b/src/SourceReflowSmoke.idric new file mode 100644 index 00000000..41d0c7d9 --- /dev/null +++ b/src/SourceReflowSmoke.idric @@ -0,0 +1,114 @@ +module SourceReflowSmoke + +import Data.Text +import IB.Reflow +import IB.Reflow.Source +import IB.Reflow.SourceBreaks +import System + +codepoint_width : Text → Number +codepoint_width value = length (unpack value) + +display_lines : List SourceLine → List Text +display_lines [] = [] +display_lines (line :: rest) = + line.visible_text :: display_lines rest + +matches : List Text → Either Text (List SourceLine) → Bool +matches wanted (Right lines) = wanted ≟ display_lines lines +matches wanted _ = False + +refused : Either Text (List SourceLine) → Bool +refused (Left _) = True +refused _ = False + +join_raw : Text → List SourceLine → Text +join_raw source [] = "" +join_raw source (line :: rest) = + source_for_line source line ++ join_raw source rest + +contiguous : Text → Number → List SourceLine → Bool +contiguous source offset [] = offset ≟ length (unpack source) +contiguous source offset (line :: rest) = + line.first_codepoint ≟ offset && + line.after_last_codepoint > offset && + line.after_last_codepoint <= length (unpack source) && + contiguous source line.after_last_codepoint rest + +source_unchanged : Text → Either Text (List SourceLine) → Bool +source_unchanged source (Right lines) = + join_raw source lines ≟ source && + contiguous source 0 lines +source_unchanged source _ = False + +kind_is : BreakKind → BreakKind → Bool +kind_is PossibleBreak PossibleBreak = True +kind_is MandatoryBreak MandatoryBreak = True +kind_is DiscretionaryHyphen DiscretionaryHyphen = True +kind_is ZeroWidthBreak ZeroWidthBreak = True +kind_is SourceEnd SourceEnd = True +kind_is _ _ = False + +has_cut : Number → BreakKind → List SourceBreak → Bool +has_cut offset wanted [] = False +has_cut offset wanted (cut :: rest) = + (offset ≟ cut.source_offset && kind_is wanted cut.break_kind) || + has_cut offset wanted rest + +some_breaks_are_refused : Bool +some_breaks_are_refused = + let source = "a\r\nb" in + let no_mandatory = [At 4 SourceEnd] in + let inside_crlf = [At 2 PossibleBreak, At 3 MandatoryBreak, At 4 SourceEnd] in + let duplicate = [At 2 PossibleBreak, At 2 PossibleBreak, At 3 SourceEnd] in + not (valid_source_breaks source no_mandatory) && + not (valid_source_breaks source inside_crlf) && + not (valid_source_breaks "a b" duplicate) && + not (valid_source_breaks "éx" [At 1 PossibleBreak, At 3 SourceEnd]) && + not (valid_source_breaks "👩‍💻x" [At 2 PossibleBreak, At 4 SourceEnd]) + +all_modes_preserve : Text → List reflow_adverb → Bool +all_modes_preserve source [] = True +all_modes_preserve source (mode :: rest) = + source_unchanged source + (reflow_source (Requirements 2 mode) codepoint_width source) && + all_modes_preserve source rest + +run_cases : Bool +run_cases = + let basic = matches ["red", "blue"] (reflow_source (Requirements 4 promptly) codepoint_width "red blue") in + let soft = "ab­cd" in + let soft_short = matches ["ab-", "cd"] (reflow_source (Requirements 3 promptly) codepoint_width soft) in + let soft_long = matches ["abcd"] (reflow_source (Requirements 4 evenly) codepoint_width soft) in + let soft_source = source_unchanged soft (reflow_source (Requirements 3 promptly) codepoint_width soft) in + let zero_width_space = matches ["ab", "cd"] (reflow_source (Requirements 2 promptly) codepoint_width "ab​cd") in + let han = matches ["漢字", "漢字"] (reflow_source (Requirements 2 evenly) codepoint_width "漢字漢字") in + let crlf = matches ["a", "b"] (reflow_source (Requirements 1 promptly) codepoint_width "a\r\nb") in + let crlf_source = source_unchanged "a\r\nb" (reflow_source (Requirements 1 promptly) codepoint_width "a\r\nb") in + let blank_line = matches ["a", "", "b"] (reflow_source (Requirements 1 promptly) codepoint_width "a\n\nb") in + let nbspace = matches ["a b", "c"] (reflow_source (Requirements 3 promptly) codepoint_width "a b c") in + let right_to_left = matches ["שלום", "hello"] (reflow_source (Requirements 5 promptly) codepoint_width "שלום hello") in + let final_newline = matches ["end"] (reflow_source (Requirements 5 promptly) codepoint_width "end\n") in + let no_break_before_joiner = not (has_cut 2 PossibleBreak (source_breaks "a ⁠b")) in + let correct_kinds = + has_cut 3 MandatoryBreak (source_breaks "a\r\nb") && + has_cut 3 DiscretionaryHyphen (source_breaks "ab­cd") && + has_cut 3 ZeroWidthBreak (source_breaks "ab​cd") in + let preserves_every_mode = all_modes_preserve "a\r\n漢字 ab­cd" [promptly, evenly, minimizing_raggedness, ignoring_final_line, minimizing_worst_gap, without_extra_lines] in + let unsupported_modes = + refused (reflow_source (Requirements 5 typographically) codepoint_width "aa bb") && + refused (reflow_source (Requirements 5 streamingly) codepoint_width "aa bb") in + let bad_width = refused (reflow_source (Requirements 0 promptly) codepoint_width "abc") in + let bad_breaks = refused (reflow_at_breaks (Requirements 5 promptly) codepoint_width "a\r\nb" [At 4 SourceEnd]) in + let empty_input = matches [] (reflow_source (Requirements 5 promptly) codepoint_width "") in + basic && soft_short && soft_long && soft_source && zero_width_space && + han && crlf && crlf_source && blank_line && nbspace && right_to_left && + final_newline && no_break_before_joiner && correct_kinds && + preserves_every_mode && unsupported_modes && bad_width && bad_breaks && + empty_input && some_breaks_are_refused + +main : IO () +main = do + let passed = run_cases + putStrLn ("source-reflow-unicode-offset-cases=" ++ show passed) + if passed then pure () else exitFailure From b43c1b890c2b03743cfdb8d6d3171a28324faf92 Mon Sep 17 00:00:00 2001 From: i Date: Mon, 26 Oct 2026 21:32:27 -0400 Subject: [PATCH 55/55] Decode retained HTML bytes without changing original source admission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the bounded Idriç windows-1252 decoder and native Grease source-bound derivative publication. Preserve original evidence and grant no reader admission. Qualify all octets, actual capture, source/process refusals, compiled mutants and independent publication controls. --- README.md | 1 + bin/ib_decode_html_bytes.grease | 129 ++++++++++++++++ docs/source-byte-decoding.md | 95 ++++++++++++ .../SourceDecodingChecks.idric | 131 ++++++++++++++++ .../source-byte-decoding/attempt.md | 145 ++++++++++++++++++ .../source-byte-decoding/host-evidence.tsv | 35 +++++ .../initial/MissingDigitLookup.idric | 10 ++ .../initial/MoonVerify.grease | 91 +++++++++++ .../initial/SourceDecodingChecks.idric | 131 ++++++++++++++++ .../source-byte-decoding/moon-host.md | 60 ++++++++ .../source-byte-decoding/moon-host.tsv | 31 ++++ .../source-byte-decoding/moon-verify.grease | 92 +++++++++++ src/IB/SourceDecoding.idric | 98 ++++++++++++ src/SourceDecodingCommand.idric | 85 ++++++++++ tests/fixtures/index-windows-1252.txt | 134 ++++++++++++++++ tests/source-decoding-mutations.grease | 40 +++++ tests/source-decoding.grease | 130 ++++++++++++++++ 17 files changed, 1438 insertions(+) create mode 100644 bin/ib_decode_html_bytes.grease create mode 100644 docs/source-byte-decoding.md create mode 100644 examples/autogenerated/source-byte-decoding/SourceDecodingChecks.idric create mode 100644 examples/autogenerated/source-byte-decoding/attempt.md create mode 100644 examples/autogenerated/source-byte-decoding/host-evidence.tsv create mode 100644 examples/autogenerated/source-byte-decoding/initial/MissingDigitLookup.idric create mode 100644 examples/autogenerated/source-byte-decoding/initial/MoonVerify.grease create mode 100644 examples/autogenerated/source-byte-decoding/initial/SourceDecodingChecks.idric create mode 100644 examples/autogenerated/source-byte-decoding/moon-host.md create mode 100644 examples/autogenerated/source-byte-decoding/moon-host.tsv create mode 100644 examples/autogenerated/source-byte-decoding/moon-verify.grease create mode 100644 src/IB/SourceDecoding.idric create mode 100644 src/SourceDecodingCommand.idric create mode 100644 tests/fixtures/index-windows-1252.txt create mode 100644 tests/source-decoding-mutations.grease create mode 100644 tests/source-decoding.grease diff --git a/README.md b/README.md index 3356f16b..bd90c72b 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ heap, form, and reconstruction outcomes separately. See ## Design notes - `docs/architecture.md` — ownership and replaceable-service boundaries +- `docs/source-byte-decoding.md` — source-bound HTML decoding without changing admission - `docs/personal-workbench.md` — personal scope, task frontend, user stories, and latency targets - `docs/prefetch-and-reading.md` — durable investigation frontiers, disposable fetches, and `~/reading` - `docs/tab-categorization.md` — overlapping personal categories and adaptive refinement diff --git a/bin/ib_decode_html_bytes.grease b/bin/ib_decode_html_bytes.grease new file mode 100644 index 00000000..66cb8024 --- /dev/null +++ b/bin/ib_decode_html_bytes.grease @@ -0,0 +1,129 @@ +#!/usr/bin/env grease +# Source-bound diagnostic decoding, not a canonical or reader-admission path. +umask 077 +var owned_temporaries = [] +trap --add EXIT { + for temporary in (owned_temporaries) { rm -rf -- $temporary } +} + +proc fail(message) { + write -- "ib source decoding: $message" >&2 + exit 2 +} + +proc checked_path(path) { + if (not path.startsWith('/')) { fail 'path must be absolute' } + var current = path + while (current !== '/') { + if test -L $current { fail 'path crosses a symlink' } + setvar current = $(dirname -- $current) + } +} + +proc regular_file(path, limit) { + checked_path $path + if test ! -f $path { fail 'regular input file unavailable' } + var count = $(wc -c < $path) + if (int(count) > limit) { fail 'file exceeds decoding bound' } +} + +proc digest(path) { + sha256sum -- $path | cut -d ' ' -f 1 +} + +proc field(path, key) { + awk -F b'\t' -v key=$key '$1 == key { print $2 }' $path +} + +if (len(ARGV) !== 4) { + fail 'usage: ib_decode_html_bytes.grease IDRIC_DECODER SOURCE.tsv RAW_BYTES NEW_DERIVATIVE_DIRECTORY' +} +var decoder = ARGV[0] +var supplied_metadata = ARGV[1] +var supplied_raw = ARGV[2] +var destination = ARGV[3] +checked_path $decoder +if test ! -x $decoder { fail 'compiled Idriç decoder required' } +regular_file $supplied_metadata 8192 +regular_file $supplied_raw 4194304 +checked_path $destination +if test -e $destination { fail 'derivative destination already exists' } +var parent = $(dirname -- $destination) +if test ! -d $parent { fail 'derivative parent unavailable' } +var stage = $(mktemp -d "$parent/.ib-decode.XXXXXX") +call owned_temporaries->append(stage) + +# Freeze bounded inputs before interpreting claims or invoking the decoder. +# The output retains this descriptor exactly; extra source fields are opaque +# provenance, not an extension to this command's admission authority. +head -c 8193 -- $supplied_metadata > "$stage/source.tsv" +head -c 4194305 -- $supplied_raw > "$stage/input.raw" +regular_file "$stage/source.tsv" 8192 +regular_file "$stage/input.raw" 4194304 +if ! awk -F b'\t' ' + BEGIN { + split("source_key source_generation sha256 byte_count charset navigation_admitted", keys, " ") + for (i in keys) required[keys[i]]=1 + } + NF!=2 || $1 !~ /^[a-z_][a-z_0-9]*$/ || $2=="" || /\r/ || seen[$1]++ {bad=1} + END {for (key in required) if (!seen[key]) bad=1; exit bad} +' "$stage/source.tsv" { fail 'malformed or incomplete source descriptor' } +var key = $(field "$stage/source.tsv" source_key) +var generation = $(field "$stage/source.tsv" source_generation) +if (len(key) > 256 or len(generation) > 256) { fail 'source identity exceeds bound' } +var expected_hash = $(field "$stage/source.tsv" sha256) +var expected_count = $(field "$stage/source.tsv" byte_count) +try { printf '%s\t%s\n' $expected_hash $expected_count | awk -F b'\t' ' + $1 !~ /^[0-9a-f]+$/ || length($1)!=64 || + $2 !~ /^(0|[1-9][0-9]*)$/ || length($2)>7 || $2>4194304 {exit 1} +' } +if failed { fail 'invalid source digest or byte count' } +var navigation = $(field "$stage/source.tsv" navigation_admitted) +if (navigation !== 'YES' and navigation !== 'NO') { fail 'invalid original navigation observation' } +var actual_count = $(wc -c < "$stage/input.raw" | tr -d '[:space:]') +var actual_hash = $(digest "$stage/input.raw") +if (actual_count !== expected_count) { fail 'source byte count mismatch' } +if (actual_hash !== expected_hash) { fail 'source digest mismatch' } +var charset = $(field "$stage/source.tsv" charset) +var source_metadata_hash = $(digest "$stage/source.tsv") + +try { $decoder $charset "$stage/input.raw" "$stage/decoded.utf8" > "$stage/producer.tsv" } +if failed { fail 'Idriç decoding producer refused or failed' } +regular_file "$stage/producer.tsv" 1024 +if ! awk -F b'\t' ' + BEGIN { + split("original_byte_count decoded_codepoint_count selected_encoding decoder_identity", keys, " ") + for (i in keys) required[keys[i]]=1 + } + NF!=2 || !($1 in required) || $2=="" || seen[$1]++ || /\r/ {bad=1} + END {for (key in required) if (!seen[key]) bad=1; exit bad} +' "$stage/producer.tsv" { fail 'invalid decoder result fields' } +if ($(field "$stage/producer.tsv" original_byte_count) !== expected_count or + $(field "$stage/producer.tsv" decoded_codepoint_count) !== expected_count) { + fail 'decoder count disagrees with frozen source' +} +if ($(field "$stage/producer.tsv" selected_encoding) !== 'windows-1252' or + $(field "$stage/producer.tsv" decoder_identity) !== 'whatwg-windows-1252-e56d49d9176e9a4-v1') { + fail 'unsupported decoder result identity' +} +regular_file "$stage/decoded.utf8" $[int(expected_count) * 3] +if ($(digest "$stage/input.raw") !== expected_hash) { fail 'source changed during decoding' } +if ($(digest "$stage/source.tsv") !== source_metadata_hash) { fail 'source descriptor changed during decoding' } +var output_count = $(wc -c < "$stage/decoded.utf8" | tr -d '[:space:]') +var output_hash = $(digest "$stage/decoded.utf8") +{ + printf 'schema\tib-decoded-html-v1\n' + printf 'source_key\t%s\nsource_generation\t%s\n' $key $generation + printf 'source_sha256\t%s\nsource_byte_count\t%s\n' $expected_hash $expected_count + printf 'source_descriptor_sha256\t%s\n' $source_metadata_hash + printf 'declared_charset\t%s\nselected_encoding\twindows-1252\n' $charset + printf 'decoder_identity\twhatwg-windows-1252-e56d49d9176e9a4-v1\n' + printf 'decoded_charset\tutf-8\ndecoded_byte_count\t%s\n' $output_count + printf 'decoded_codepoint_count\t%s\ndecoded_sha256\t%s\n' $expected_count $output_hash + printf 'original_navigation_admitted\t%s\nreader_admission\tNOT_GRANTED\n' $navigation +} > "$stage/decode.tsv" +digest "$stage/decode.tsv" > "$stage/complete" +rm -- "$stage/input.raw" "$stage/producer.tsv" +mv -T --no-clobber -- $stage $destination +if test -d $stage { fail 'derivative publication conflicted' } +write -- $destination diff --git a/docs/source-byte-decoding.md b/docs/source-byte-decoding.md new file mode 100644 index 00000000..b9bc2be5 --- /dev/null +++ b/docs/source-byte-decoding.md @@ -0,0 +1,95 @@ +# Decode retained bytes without changing source admission + +`IB.SourceDecoding` owns the bounded HTML byte-decoding step. Its first decoder +implements the WHATWG windows-1252 index and the 17 HTML labels which select it, +including ISO-8859-1, Latin1 and ASCII. It does not reinterpret a label based on +whether the particular bytes happen to form valid UTF-8. + +ASCII whitespace is trimmed from labels and ASCII case is ignored. NBSP is not +ASCII label whitespace. Every source octet has one Unicode scalar in this +index, including the five C1 slots sometimes treated as undefined by other +libraries. NUL, CR/LF and those C1 scalars remain present; HTML preprocessing, +recovery and extraction have different owners. + +A recognized UTF-8/UTF-16 BOM selects a different HTML decoder. This bounded +implementation explicitly refuses those BOMs rather than decoding their bytes +under a conflicting windows-1252 label. UTF-8/UTF-16 decoder support and the +full HTML encoding-prescan algorithm remain future work. + +## Binary command and disposable output + +`SourceDecodingCommand.idric` reads actual bytes using the existing binary +Buffer/file interfaces. It checks file size before allocation, requires the +actual read count to match, probes EOF, and writes actual UTF-8 bytes through +a binary buffer. Input is bounded to 4 MiB; it still holds the whole bounded +source and decoded value. This is not streaming ingestion. + +The native Grease adapter `bin/ib_decode_html_bytes.grease` takes an explicit +compiled decoder, a source descriptor TSV, raw source path and a new disposable +destination directory. Inputs and executable paths must be absolute. A source +descriptor is bounded key/value data with distinct keys and these required +fields: + +| Field | Meaning | +| --- | --- | +| `source_key` | Existing caller-supplied source reference | +| `source_generation` | Existing caller-supplied version reference | +| `sha256` | Expected original-byte SHA-256 | +| `byte_count` | Expected original-byte count, at most 4 MiB | +| `charset` | Original declared HTML encoding label | +| `navigation_admitted` | Original YES/NO observation, never changed here | + +Additional well-formed fields are opaque provenance and are retained exactly. +These are source claims supplied by the caller, not authentication or proof of +an Arena reopen, current generation or permission to navigate. + +The adapter snapshots the bounded descriptor and raw input, verifies original +count/digest, requires producer success and consistent decoder counts/identity, +and checks that neither frozen input changed during decoding. It stages beneath +the verified destination parent and publishes by a no-clobber same-filesystem +directory rename. A failed producer or conflict cannot replace an earlier +derivative. Static path checks do not promise containment against concurrent +filesystem-topology replacement or power-loss durability. + +The output directory contains: + +- `source.tsv`: the exact supplied descriptor, including the original charset + and navigation observation; +- `decoded.utf8`: disposable derived bytes; +- `decode.tsv`: source and descriptor hashes, decoder identity, selected + encoding, output hash/count, decoded codepoint count and explicit + `reader_admission=NOT_GRANTED`; +- `complete`: the derived metadata digest. + +The raw body is not duplicated into another persistent store. Byte offsets in +the original source and decoded Unicode codepoint offsets are different units. +The original `HtmlEvidence`, native source owner and Arena admission paths are +unchanged; this receipt is not one of their accepted admission schemas. + +## Specification and acceptance + +The independent normative index fixture retains WHATWG identifier +`e56d49d9176e9a412283cf29ac9bd613f5620462f2a080a84eceaf974cfa18b7` +(2024-09-18) from: + +- https://encoding.spec.whatwg.org/#names-and-labels +- https://encoding.spec.whatwg.org/#single-byte-decoder +- https://encoding.spec.whatwg.org/index-windows-1252.txt +- https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding + +`SourceDecodingChecks.idric` checks all 256 octets against that fixture, label +rules, special scalars and BOM refusals. `tests/source-decoding.grease` executes +the real binary command and publication adapter, checks bad source claims and +producer failures, and consumes the exact existing Google capture supplied by +absolute path. `tests/source-decoding-mutations.grease` compiles and rejects +actual naïve-Latin1 and undefined-byte-replacement decoders against the same +normative oracle. These are host decoder/process results, not Android evidence. + +The retained Google capture from search source +`dcbd86c0f7a69c4e0b381c0ab53f7c41f9966900` has 93,166 ASCII bytes, SHA-256 +`f71e991ab87271e77255fe2d2454cb4cb401fe47cdf953ea550c6e0fcd8afeb4`. +Its derivative is therefore byte-identical, while the declared charset remains +ISO-8859-1 and redirect count remains one. The test descriptor explicitly uses +a fixture identity, not an invented live Arena generation. The missing redirect +chain and native reader/DEX connection still prevent joined navigation +acceptance. Decoding does not make those conditions true. diff --git a/examples/autogenerated/source-byte-decoding/SourceDecodingChecks.idric b/examples/autogenerated/source-byte-decoding/SourceDecodingChecks.idric new file mode 100644 index 00000000..f17b3d84 --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/SourceDecodingChecks.idric @@ -0,0 +1,131 @@ +module SourceDecodingChecks + +import Data.Text +import Data.List +import Data.Buffer +import System +import System.File +import System.File.Buffer +import IB.SourceDecoding + +index_identity : Text +index_identity = "# Identifier: e56d49d9176e9a412283cf29ac9bd613f5620462f2a080a84eceaf974cfa18b7" + +position_in_digits : Char → Int → List Char → Maybe Int +position_in_digits expected position [] = Nothing +position_in_digits expected position (current :: rest) = + if current ≟ expected then Just position + else position_in_digits expected (position + 1) rest + +digit_value : Char → Maybe Int +digit_value character = position_in_digits character 0 (unpack "0123456789") + +hex_value : Char → Maybe Int +hex_value character = + case position_in_digits character 0 (unpack "0123456789abcdef") of + Just value ⇒ Just value + Nothing ⇒ position_in_digits character 0 (unpack "0123456789ABCDEF") + +digits_from : (Char → Maybe Int) → Int → Int → List Char → Maybe Int +digits_from digit base value [] = Just value +digits_from digit base value (character :: rest) = do + next ← digit character + digits_from digit base (value × base + next) rest + +decimal : Text → Maybe Int +decimal value = case unpack value of + [] ⇒ Nothing + digits ⇒ digits_from digit_value 10 0 digits + +hexadecimal : Text → Maybe Int +hexadecimal value = case unpack value of + '0' :: 'x' :: first :: rest ⇒ digits_from hex_value 16 0 (first :: rest) + _ ⇒ Nothing + +index_values : Int → List Text → Either Text (List Int) +index_values expected [] = + if expected ≟ 128 then Right [] else Left "normative index has missing entries" +index_values expected (line :: rest) = + case words line of + [] ⇒ index_values expected rest + "#" :: _ ⇒ index_values expected rest + pointer :: codepoint :: _ ⇒ + case (decimal pointer, hexadecimal codepoint) of + (Just actual, Just scalar) ⇒ + if actual ≟ expected && scalar >= 0 && scalar <= 0x10FFFF + then map (scalar ::) (index_values (expected + 1) rest) + else Left "normative index order or scalar invalid" + _ ⇒ Left "normative index numeric fields invalid" + _ ⇒ Left "normative index row invalid" + +assert_case : Text → Bool → IO () +assert_case label True = putStrLn ("PASS " ++ label) +assert_case label False = die ("FAIL " ++ label) + +decoded_scalars : Text → List SourceOctet → Maybe (List Int) +decoded_scalars label octets = + case decode_html_bytes label octets of + Left _ ⇒ Nothing + Right decoded ⇒ Just (map ord (unpack decoded.decoded_text)) + +is_refused : Text → List SourceOctet → Bool +is_refused label octets = case decode_html_bytes label octets of + Left _ ⇒ True + Right _ ⇒ False + +write_text_bytes : Text → Text → IO () +write_text_bytes path value = do + let count = stringByteLength value + Just buffer ← newBuffer count | Nothing ⇒ die "test UTF-8 allocation failed" + setString buffer 0 value + Right () ← writeBufferToFile path buffer count | Left _ ⇒ die "test UTF-8 write failed" + pure () + +covering +write_octet_fixture : Text → IO () +write_octet_fixture path = do + Just buffer ← newBuffer 256 | Nothing ⇒ die "test raw allocation failed" + fill buffer 0 + Right () ← writeBufferToFile path buffer 256 | Left _ ⇒ die "test raw write failed" + pure () + where + covering + fill : Buffer → Int → IO () + fill buffer offset = + if offset >= 256 then pure () else do + setBits8 buffer offset (cast offset) + fill buffer (offset + 1) + +covering +main : IO () +main = do + arguments ← getArgs + case arguments of + [_, index_path, all_octets_path, expected_utf8_path] ⇒ do + Right index ← readFile index_path | Left _ ⇒ die "normative index unreadable" + assert_case "normative index identity" (elem index_identity (lines index)) + Right high_scalars ← pure (index_values 0 (lines index)) | Left reason ⇒ die reason + let expected_scalars = [0..127] ++ high_scalars + let octets : List SourceOctet = map cast (the (List Int) [0..255]) + assert_case "all 256 octets match normative index" + (decoded_scalars "windows-1252" octets ≟ Just expected_scalars) + assert_case "NUL CR LF Euro quotes cafe NBSP and five C1 scalars" + (decoded_scalars "ISO-8859-1" [0,13,10,128,145,146,233,160,129,141,143,144,157] ≟ + Just [0,13,10,0x20AC,0x2018,0x2019,0xE9,0xA0,0x81,0x8D,0x8F,0x90,0x9D]) + assert_case "seventeen labels" + (all windows_1252_label ["ansi_x3.4-1968", "ascii", "cp1252", "cp819", + "csisolatin1", "ibm819", "iso-8859-1", "iso-ir-100", "iso8859-1", + "iso88591", "iso_8859-1", "iso_8859-1:1987", "l1", "latin1", + "us-ascii", "windows-1252", "x-cp1252"]) + assert_case "ASCII trim and case" (windows_1252_label "\t ISO-8859-1\r\n") + assert_case "NBSP padding is not ASCII whitespace" + (not (windows_1252_label (pack [chr 160] ++ "windows-1252"))) + assert_case "unknown label refused" (is_refused "utf-8" [65]) + assert_case "UTF-8 BOM refuses wrong decoder" (is_refused "windows-1252" [0xEF,0xBB,0xBF,65]) + assert_case "UTF-16 LE BOM refuses wrong decoder" (is_refused "windows-1252" [0xFF,0xFE,65,0]) + assert_case "UTF-16 BE BOM refuses wrong decoder" (is_refused "windows-1252" [0xFE,0xFF,0,65]) + assert_case "empty input decodes without acquiring admission" (decoded_scalars "latin1" [] ≟ Just []) + write_octet_fixture all_octets_path + write_text_bytes expected_utf8_path (pack (map chr expected_scalars)) + putStrLn "PASS source byte decoding semantic controls" + _ ⇒ die "usage: source-decoding-checks NORMATIVE_INDEX ALL_OCTETS EXPECTED_UTF8" diff --git a/examples/autogenerated/source-byte-decoding/attempt.md b/examples/autogenerated/source-byte-decoding/attempt.md new file mode 100644 index 00000000..86404322 --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/attempt.md @@ -0,0 +1,145 @@ +# Program attempt: source-byte-decoding + +Status: PASS for bounded host decoding and process checks; integration partial. + +## Purpose and intended behavior + +Decode a retained HTML source into a disposable UTF-8 representation without +changing its original bytes, declared charset, generation or navigation +admission. The current Google observation declares ISO-8859-1; HTML uses the +WHATWG windows-1252 decoder for that label. Decoding alone cannot admit its +redirected navigation or join the Android reader. + +## Type-system sketch + +The input domain is octets, not already-decoded Text. `SourceOctet` names the +existing Bits8 raw boundary. `HtmlDecodedText` contains the exact supplied +label, selected encoding, original octet count and decoded Unicode text. +`decode_html_bytes : Text → List SourceOctet → Either Text HtmlDecodedText` +is pure. The 17 windows-1252 labels use ASCII whitespace/case rules. All 256 +octets have defined mappings; CR/LF, NUL and C1 values remain unchanged where +the index specifies them. HTML preprocessing belongs to the existing recovery +owner. Recognized UTF-8/UTF-16 BOMs explicitly refuse in this bounded decoder +instead of being misread as windows-1252. + +The command uses the compiler's binary Buffer interfaces, checks a bounded file +size, the actual read count and EOF, then writes UTF-8 bytes through a binary +buffer. Input is bounded to 4 MiB; this is not a streaming-ingestion repair. +The command reports codepoint count separately from byte count. + +Grease owns source/derivative hashing, immutable input snapshots, producer exit +checking and publication of a private staged directory by same-filesystem +rename. It retains a caller-supplied source descriptor byte-for-byte, including +source key, generation, original digest/count, declared charset and original +navigation admission. Opaque extra descriptor fields are preserved as data. +The derived receipt always says `reader_admission=NOT_GRANTED`; it cannot be +used as an original UTF-8 `HtmlEvidence` or as an Arena admission receipt. + +## Specification and runtime boundary + +Primary specification: WHATWG Encoding names/labels and single-byte decoder; +windows-1252 index identifier +`e56d49d9176e9a412283cf29ac9bd613f5620462f2a080a84eceaf974cfa18b7` +(2024-09-18), retained independently as a test fixture. In particular, 0x81, +0x8D, 0x8F, 0x90 and 0x9D map to their C1 scalars; they are not replaced or +rejected. HTML's BOM precedence is preserved by explicit refusal until BOM +decoder paths exist. + +Use the verified Linux x86-64 Idriç host bundle at compiler commit +`94dfd99bd3e376507fedc8611053b7173b2519f0`, archive SHA-256 +`0be57deab9410e8d4c09c7198bf7e8ee0ec3e24121f76640147efa4a53d120da`. +No RefC, iconv, Python or Java decoder is a runtime substitute. + +## Planned acceptance + +Compile and execute actual Idriç code against all 256 bytes and the independent +normative index; alias/ASCII-whitespace and BOM refusals; NUL/CRLF/Euro/quotes/ +café controls; real captured Google body identity; bad source count/hash and +nonzero producer publication refusal. Known-bad Latin1 and replacement mutants +must fail the same oracle. Preserve original admission and metadata. + +## Result, fallback and remaining boundary + +Fallback: none. DEX/ART, +Android, canonical Arena admission, joined Prepaint and physical devices remain +NOT_RUN. The existing native redirect-provenance and charset gates are unchanged. + +## Executed result + +The production module and binary-file command compiled with the exact verified +94df host bundle. A fresh source staging directory received the current `src/` +tree plus `SourceDecodingChecks.idric`, then executed these compiler arguments: + +```text +SourceDecodingCommand.idric -o source-decoder +SourceDecodingChecks.idric -o source-decoding-checks +``` + +`bin/idric` and the execution wrapper `bin/idric-env` were the absolute +entrypoints inside that verified bundle, not ambient commands. Native Grease +was the verified installed runtime with binary SHA-256 +`7e31cd05b7a9d8fb2a4a9e003a7f3fcb0159138506d17f0fb28da8cbe22aa85c`. +Its inherited LeakSanitizer could not inspect this host's process-task data; +`ASAN_OPTIONS=detect_leaks=0` was explicit for orchestration execution. Leak +analysis is not an accepted result. + +Actual execution of the compiled checks and `tests/source-decoding.grease` +passed with exit zero. The normative all-octet check matched all 256 scalars. +The binary command produced exactly 401 UTF-8 bytes from the 256-octet fixture; +independent byte checks retained the entire 128-byte ASCII prefix (including +NUL and CR/LF) and verified Euro/C1, NBSP and café UTF-8 octets. Alias, ASCII +trimming, BOM, unknown-label, size-bound, empty-input and input-as-output +controls passed. + +Native process tests refused wrong source count/hash, a nonzero producer with +partial output, and a producer which changed the frozen descriptor. No accepted +derivative appeared; the earlier valid derivative remained exact. The real +93,166-byte Google observation round-tripped byte-for-byte; its original +ISO-8859-1 and redirect-count-one metadata were unchanged and the derived +receipt still grants no reader admission. The test uses an explicit fixture +identity rather than inventing a live Arena generation. + +`tests/source-decoding-mutations.grease` compiled two actual changed Idriç +decoders in distinct temporary source roots. The naïve Latin1 decoder and the +five-C1-slot replacement decoder both failed the same normative all-256 scalar +assertion with nonzero exits. The real decoder passed that assertion. + +The final fresh command and check-program native `.so` hashes, source hashes, +normative fixture and bounded outcomes are in `host-evidence.tsv`. Vocabulary +and whitespace checks passed. Independent qualification normalized the fixture +to the exact fetched primary bytes by removing an additional trailing newline, +then reran the semantic, process and compiled-mutant checks. Its digest is +`2c5fa192d566f92b8b9e6e116255b1f94016b4ece5b9dacb3b98be70b8c60795`; +the index identifier and every numeric mapping are unchanged. See +`moon-host.md` for the independent rebuild and publication controls. +Existing `HtmlEvidence`, canonical-store and native reader admission source +files are unchanged. + +The continuation recheck compared the maintained module, command and semantic +check source byte-for-byte with the qualified staging tree, verified the +recorded native program digests, then re-executed the full semantic/process +controls and both freshly compiled mutant checks. They again passed with exit +zero on this Ubuntu 24.04 x86-64 host. This repeated execution does not upgrade +the Android, Arena or joined-navigation evidence boundary. + +## Earlier test-helper failures and language work + +The initial style-adjusted helper used Unicode `−`, which this 94df bundle +parsed but could not resolve. Its source is retained under `initial/`. A +subsequent reduced lookup attempt exposed that this pinned `Data.List` does not +export the assumed `elemIndex`; that rejected assumption is also retained. +The maintained helper uses a simple descriptive digit-position lookup, still +entirely in Idriç. No external decoder or language substitution was introduced. + +Those compiler failures were not counted as acceptance. Final verification +rebuilt from a fresh staging root and required successful compilation before +running the new executable; earlier generated executables were not carried +forward as final-head proof. + +The first remaining integration boundary is a verified decoded representation +consumer which retains original byte identity and independently establishes +reader admission. This diagnostic does not remove the missing redirect chain, +implement bounded streaming, change the native original-UTF8 gate or execute +the decoder under DEX/ART. The language bundle's missing canonical subtraction +binding remains a separately identified compatibility gap; no language change +was necessary to deliver this decoder slice. diff --git a/examples/autogenerated/source-byte-decoding/host-evidence.tsv b/examples/autogenerated/source-byte-decoding/host-evidence.tsv new file mode 100644 index 00000000..822621c1 --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/host-evidence.tsv @@ -0,0 +1,35 @@ +schema ib-source-decoding-host-evidence-v1 +scope HOST_IDRIC_BYTES_AND_NATIVE_GREASE +ib_starting_source ff8423697ef1d25140f4e77594b224ce7b87fef7 +compiler_source 94dfd99bd3e376507fedc8611053b7173b2519f0 +compiler_archive_sha256 0be57deab9410e8d4c09c7198bf7e8ee0ec3e24121f76640147efa4a53d120da +grease_binary_sha256 7e31cd05b7a9d8fb2a4a9e003a7f3fcb0159138506d17f0fb28da8cbe22aa85c +decoder_module_sha256 e3b81a3450878f23a9c5228ca6695e64692bae4367560e5abdabe6adaed327f5 +decoder_command_source_sha256 bb81e9512d5c211af777d7e0b5db978a806ce951410724693b2494153608d9ab +decoder_native_program_sha256 8912f612d40b97845fc726a38064a3e606c92cec7099b4d86f420dbef4003d16 +semantic_check_native_program_sha256 66e4be6de322ad6972156b4fae3ac5e62a417d1221506119d0a54e6bfbfa72e5 +normative_index_sha256 2c5fa192d566f92b8b9e6e116255b1f94016b4ece5b9dacb3b98be70b8c60795 +all_256_scalars PASS +all_octet_utf8_bytes 401 +binary_NUL_CR_LF_C1 PASS +seventeen_labels_ASCII_trim_case PASS +NBSP_label_padding REFUSED +recognized_conflicting_BOMs REFUSED +wrong_source_count_hash REFUSED_NO_DERIVATIVE +nonzero_partial_producer REFUSED_NO_DERIVATIVE +changed_frozen_descriptor REFUSED_NO_DERIVATIVE +naive_Latin1_mutant REJECTED_NORMATIVE_SCALAR_ORACLE +undefined_byte_replacement_mutant REJECTED_NORMATIVE_SCALAR_ORACLE +actual_google_body_bytes 93166 +actual_google_body_sha256 f71e991ab87271e77255fe2d2454cb4cb401fe47cdf953ea550c6e0fcd8afeb4 +actual_google_derivative BYTE_IDENTICAL +actual_google_declared_charset ISO-8859-1_UNCHANGED +actual_google_redirect_count 1_UNCHANGED +derived_reader_admission NOT_GRANTED +existing_HtmlEvidence_native_Arena_gates UNCHANGED +streaming_ingestion NOT_IMPLEMENTED +joined_Prepaint NOT_RUN +DEX_ART NOT_RUN +Android_Binder NOT_RUN +physical_C67 NOT_RUN +physical_A1 NOT_RUN diff --git a/examples/autogenerated/source-byte-decoding/initial/MissingDigitLookup.idric b/examples/autogenerated/source-byte-decoding/initial/MissingDigitLookup.idric new file mode 100644 index 00000000..ecb0540f --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/initial/MissingDigitLookup.idric @@ -0,0 +1,10 @@ +module MissingDigitLookup + +import Data.Text +import Data.List + +-- Reduced initial test-helper attempt: the pinned 94df library does not export +-- elemIndex through Data.List. The maintained helper uses an explicit bounded +-- position lookup instead; this file records the rejected library assumption. +digit_value : Char → Maybe Int +digit_value character = map cast (elemIndex character (unpack "0123456789")) diff --git a/examples/autogenerated/source-byte-decoding/initial/MoonVerify.grease b/examples/autogenerated/source-byte-decoding/initial/MoonVerify.grease new file mode 100644 index 00000000..2aec8ab5 --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/initial/MoonVerify.grease @@ -0,0 +1,91 @@ +#!/usr/bin/env grease +# Independent source-decoder rebuild and publication/admission controls. +umask 077 +if (len(ARGV) !== 6) { + write -- 'usage: moon-verify.grease IDRIC GREASE GOOGLE_BODY GOOGLE_METADATA PRIMARY_INDEX NEW_EVIDENCE' >&2 + exit 2 +} +var compiler = ARGV[0] +var grease = ARGV[1] +var google = ARGV[2] +var google_metadata = ARGV[3] +var primary_index = ARGV[4] +var evidence = ARGV[5] +for path in (ARGV) { + if (not path.startsWith('/')) { write -- 'FAIL absolute input required' >&2; exit 2 } +} +if test -e $evidence { write -- 'FAIL refusing existing evidence' >&2; exit 2 } +var example = $(dirname -- $0) +var root = $(realpath -- "$example/../../..") +var source = "$evidence/source" +mkdir -p $source +cp -R "$root/src/." "$source/" +cp "$example/SourceDecodingChecks.idric" "$source/" +sha256sum "$root/src/IB/SourceDecoding.idric" "$root/src/SourceDecodingCommand.idric" $primary_index > "$evidence/inputs.sha256" +git -C $root rev-parse HEAD > "$evidence/source-ref.txt" +var previous = $(pwd) +cd $source +$compiler SourceDecodingCommand.idric -o source-decoder > "$evidence/command-compile.log" 2>&1 +$compiler SourceDecodingChecks.idric -o source-decoding-checks > "$evidence/checks-compile.log" 2>&1 +cd $previous +if rg -n '^[[:space:]]*(Error:|INTERNAL ERROR)' "$evidence/command-compile.log" "$evidence/checks-compile.log" { + write -- 'FAIL fresh compiler diagnostic' >&2 + exit 1 +} +var decoder = "$source/build/exec/source-decoder" +var checks = "$source/build/exec/source-decoding-checks" +sha256sum "${decoder}_app/source-decoder.so" "${checks}_app/source-decoding-checks.so" > "$evidence/programs.sha256" +$checks $primary_index "$evidence/all.raw" "$evidence/expected.utf8" > "$evidence/primary-index.log" +env GREASE=$grease $grease "$root/tests/source-decoding.grease" $checks $decoder $google $google_metadata > "$evidence/process.log" 2>&1 +env IDRIC=$compiler $grease "$root/tests/source-decoding-mutations.grease" > "$evidence/mutants.log" 2>&1 + +var raw = "$evidence/all.raw" +var descriptor = "$evidence/source.tsv" +var hash = $(sha256sum $raw | cut -d ' ' -f 1) +printf 'source_key\tmoon:source-generation\nsource_generation\t42-retained\nsha256\t%s\nbyte_count\t256\ncharset\tISO-8859-1\nnavigation_admitted\tYES\nredirect_count\t1\nopaque_origin\tretained provenance with spaces\n' $hash > $descriptor +$grease "$root/bin/ib_decode_html_bytes.grease" $decoder $descriptor $raw "$evidence/yes-derived" > "$evidence/yes-result.log" 2>&1 +cmp $descriptor "$evidence/yes-derived/source.tsv" +cmp "$evidence/expected.utf8" "$evidence/yes-derived/decoded.utf8" +grep -Fx b'original_navigation_admitted\tYES' "$evidence/yes-derived/decode.tsv" +grep -Fx b'reader_admission\tNOT_GRANTED' "$evidence/yes-derived/decode.tsv" +grep -Fx b'opaque_origin\tretained provenance with spaces' "$evidence/yes-derived/source.tsv" +printf '\033[32mPASS\033[0m source navigation YES and opaque provenance preserved without reader admission\n' + +cp $descriptor "$evidence/duplicate.tsv" +printf 'charset\tISO-8859-1\n' >> "$evidence/duplicate.tsv" +try { $grease "$root/bin/ib_decode_html_bytes.grease" $decoder "$evidence/duplicate.tsv" $raw "$evidence/duplicate-derived" > "$evidence/duplicate.log" 2>&1 } +if (not _error.code) { write -- 'FAIL duplicate descriptor accepted' >&2; exit 1 } +test ! -e "$evidence/duplicate-derived" +grep -F 'malformed or incomplete source descriptor' "$evidence/duplicate.log" +sed '/^source_generation\t/d' $descriptor > "$evidence/missing.tsv" +try { $grease "$root/bin/ib_decode_html_bytes.grease" $decoder "$evidence/missing.tsv" $raw "$evidence/missing-derived" > "$evidence/missing.log" 2>&1 } +if (not _error.code) { write -- 'FAIL missing source generation accepted' >&2; exit 1 } +test ! -e "$evidence/missing-derived" +grep -F 'malformed or incomplete source descriptor' "$evidence/missing.log" +printf '\033[32mPASS\033[0m malformed descriptor refusals have no derivative\n' + +# A deterministic process fixture creates the winner after initial precheck. +printf '#!%s\n' $grease > "$evidence/competing-producer" +cat >> "$evidence/competing-producer" <<'EOF' +var decoder = ENV.MOON_REAL_DECODER +$decoder @ARGV +var destination = ENV.MOON_COMPETING_DESTINATION +mkdir $destination +printf 'retained winner\n' > "$destination/winner" +EOF +chmod +x "$evidence/competing-producer" +try { + env MOON_REAL_DECODER=$decoder MOON_COMPETING_DESTINATION="$evidence/competing-derived" \ + $grease "$root/bin/ib_decode_html_bytes.grease" "$evidence/competing-producer" $descriptor $raw "$evidence/competing-derived" > "$evidence/competing.log" 2>&1 +} +if (not _error.code) { write -- 'FAIL competing winner was reported as our publication' >&2; exit 1 } +grep -F 'derivative publication conflicted' "$evidence/competing.log" +test $(cat "$evidence/competing-derived/winner") = 'retained winner' +test ! -e "$evidence/competing-derived/source.tsv" +test ! -e "$evidence/competing-derived/decode.tsv" +var leftovers = $(find $evidence -maxdepth 1 -name '.ib-decode.*' -print) +if (leftovers !== '') { write -- 'FAIL discarded staging directory survived' >&2; exit 1 } +cmp $descriptor "$evidence/yes-derived/source.tsv" +cmp "$evidence/expected.utf8" "$evidence/yes-derived/decoded.utf8" +printf '\033[32mPASS\033[0m competing winner and previous derivative survive; losing stage removed\n' +printf 'fresh_Idric_build\tPASS\nprimary_index_all256\tPASS\nprovided_process_controls\tPASS\ncompiled_mutants_rejected\tPASS\noriginal_navigation_YES\tPRESERVED\nopaque_provenance\tPRESERVED\nreader_admission\tNOT_GRANTED\nmalformed_descriptor\tREFUSED\ncompeting_winner\tPRESERVED\nlosing_stage\tREMOVED\nART\tNOT_RUN\njoined_Prepaint\tNOT_RUN\nphysical_device\tNOT_RUN\n' > "$evidence/result.tsv" diff --git a/examples/autogenerated/source-byte-decoding/initial/SourceDecodingChecks.idric b/examples/autogenerated/source-byte-decoding/initial/SourceDecodingChecks.idric new file mode 100644 index 00000000..37440c8d --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/initial/SourceDecodingChecks.idric @@ -0,0 +1,131 @@ +module SourceDecodingChecks + +import Data.Text +import Data.List +import Data.Buffer +import System +import System.File +import System.File.Buffer +import IB.SourceDecoding + +index_identity : Text +index_identity = "# Identifier: e56d49d9176e9a412283cf29ac9bd613f5620462f2a080a84eceaf974cfa18b7" + +digit_value : Char → Maybe Int +digit_value character = + let scalar = ord character in + if scalar >= 48 && scalar <= 57 then Just (scalar − 48) else Nothing + +hex_value : Char → Maybe Int +hex_value character = + case digit_value character of + Just value ⇒ Just value + Nothing ⇒ + let scalar = ord character in + if scalar >= 65 && scalar <= 70 then Just (scalar − 55) + else if scalar >= 97 && scalar <= 102 then Just (scalar − 87) + else Nothing + +digits_from : (Char → Maybe Int) → Int → Int → List Char → Maybe Int +digits_from digit base value [] = Just value +digits_from digit base value (character :: rest) = do + next ← digit character + digits_from digit base (value × base + next) rest + +decimal : Text → Maybe Int +decimal value = case unpack value of + [] ⇒ Nothing + digits ⇒ digits_from digit_value 10 0 digits + +hexadecimal : Text → Maybe Int +hexadecimal value = case unpack value of + '0' :: 'x' :: first :: rest ⇒ digits_from hex_value 16 0 (first :: rest) + _ ⇒ Nothing + +index_values : Int → List Text → Either Text (List Int) +index_values expected [] = + if expected ≟ 128 then Right [] else Left "normative index has missing entries" +index_values expected (line :: rest) = + case words line of + [] ⇒ index_values expected rest + "#" :: _ ⇒ index_values expected rest + pointer :: codepoint :: _ ⇒ + case (decimal pointer, hexadecimal codepoint) of + (Just actual, Just scalar) ⇒ + if actual ≟ expected && scalar >= 0 && scalar <= 0x10FFFF + then map (scalar ::) (index_values (expected + 1) rest) + else Left "normative index order or scalar invalid" + _ ⇒ Left "normative index numeric fields invalid" + _ ⇒ Left "normative index row invalid" + +assert_case : Text → Bool → IO () +assert_case label True = putStrLn ("PASS " ++ label) +assert_case label False = die ("FAIL " ++ label) + +decoded_scalars : Text → List SourceOctet → Maybe (List Int) +decoded_scalars label octets = + case decode_html_bytes label octets of + Left _ ⇒ Nothing + Right decoded ⇒ Just (map ord (unpack decoded.decoded_text)) + +is_refused : Text → List SourceOctet → Bool +is_refused label octets = case decode_html_bytes label octets of + Left _ ⇒ True + Right _ ⇒ False + +write_text_bytes : Text → Text → IO () +write_text_bytes path value = do + let count = stringByteLength value + Just buffer ← newBuffer count | Nothing ⇒ die "test UTF-8 allocation failed" + setString buffer 0 value + Right () ← writeBufferToFile path buffer count | Left _ ⇒ die "test UTF-8 write failed" + pure () + +covering +write_octet_fixture : Text → IO () +write_octet_fixture path = do + Just buffer ← newBuffer 256 | Nothing ⇒ die "test raw allocation failed" + fill buffer 0 + Right () ← writeBufferToFile path buffer 256 | Left _ ⇒ die "test raw write failed" + pure () + where + covering + fill : Buffer → Int → IO () + fill buffer offset = + if offset >= 256 then pure () else do + setBits8 buffer offset (cast offset) + fill buffer (offset + 1) + +covering +main : IO () +main = do + arguments ← getArgs + case arguments of + [_, index_path, all_octets_path, expected_utf8_path] ⇒ do + Right index ← readFile index_path | Left _ ⇒ die "normative index unreadable" + assert_case "normative index identity" (elem index_identity (lines index)) + Right high_scalars ← pure (index_values 0 (lines index)) | Left reason ⇒ die reason + let expected_scalars = [0..127] ++ high_scalars + let octets : List SourceOctet = map cast (the (List Int) [0..255]) + assert_case "all 256 octets match normative index" + (decoded_scalars "windows-1252" octets ≟ Just expected_scalars) + assert_case "NUL CR LF Euro quotes cafe NBSP and five C1 scalars" + (decoded_scalars "ISO-8859-1" [0,13,10,128,145,146,233,160,129,141,143,144,157] ≟ + Just [0,13,10,0x20AC,0x2018,0x2019,0xE9,0xA0,0x81,0x8D,0x8F,0x90,0x9D]) + assert_case "seventeen labels" + (all windows_1252_label ["ansi_x3.4-1968", "ascii", "cp1252", "cp819", + "csisolatin1", "ibm819", "iso-8859-1", "iso-ir-100", "iso8859-1", + "iso88591", "iso_8859-1", "iso_8859-1:1987", "l1", "latin1", + "us-ascii", "windows-1252", "x-cp1252"]) + assert_case "ASCII trim and case" (windows_1252_label "\t ISO-8859-1\r\n") + assert_case "NBSP padding is not ASCII whitespace" + (not (windows_1252_label (pack [chr 160] ++ "windows-1252"))) + assert_case "unknown label refused" (is_refused "utf-8" [65]) + assert_case "UTF-8 BOM refuses wrong decoder" (is_refused "windows-1252" [0xEF,0xBB,0xBF,65]) + assert_case "UTF-16 LE BOM refuses wrong decoder" (is_refused "windows-1252" [0xFF,0xFE,65,0]) + assert_case "UTF-16 BE BOM refuses wrong decoder" (is_refused "windows-1252" [0xFE,0xFF,0,65]) + assert_case "empty input decodes without acquiring admission" (decoded_scalars "latin1" [] ≟ Just []) + write_octet_fixture all_octets_path + write_text_bytes expected_utf8_path (pack (map chr expected_scalars)) + putStrLn "PASS source byte decoding semantic controls" + _ ⇒ die "usage: source-decoding-checks NORMATIVE_INDEX ALL_OCTETS EXPECTED_UTF8" diff --git a/examples/autogenerated/source-byte-decoding/moon-host.md b/examples/autogenerated/source-byte-decoding/moon-host.md new file mode 100644 index 00000000..1dbb59b6 --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/moon-host.md @@ -0,0 +1,60 @@ +# Independent host qualification + +Prerequisite: decoder implementation commit +`f6c1c195f71070014e7aa5b254beea20051f91e5`, tree +`364684f10d4cfa95f5996a6c6cdb68b21d686955`. + +The settled semantics are an octet-to-Unicode decoding operation and a +source-bound disposable derivative. Original navigation observations are +preserved as data; derivative creation grants no reader admission, including +when the original observation says YES. Grease owns process orchestration and +same-filesystem no-clobber publication. The actual Idriç binary owns decoding. + +The independent check rebuilds the existing Idriç command and semantic oracle +in a fresh source root, runs the provided controls and compiled mutants, then +checks navigation-YES preservation, opaque descriptor retention, malformed +descriptor refusal and a competing destination created after the initial +precheck. The competing winner must survive and the staged loser must be +removed. No decoder semantics or native admission gate is changed. + +The primary WHATWG index was fetched independently from +https://encoding.spec.whatwg.org/index-windows-1252.txt. It has identifier +`e56d49d9176e9a412283cf29ac9bd613f5620462f2a080a84eceaf974cfa18b7`, +SHA-256 `2c5fa192d566f92b8b9e6e116255b1f94016b4ece5b9dacb3b98be70b8c60795` +and 5,667 bytes. The initial retained fixture had one additional trailing +newline; `diff -u` showed only that final blank line. The fixture is now exactly +byte-identical to the independently fetched primary file. Its index identifier +and every numeric mapping are unchanged. The fresh command must also pass +against the fetched file. + +The first supplemental harness expected the adapter's post-rename conflict +message. This host's `mv --no-clobber` refuses with exit 1 and “not replacing” +before that message; the winner survived and the losing stage was removed. +That failed harness is retained under `initial/MoonVerify.grease`. The updated +check accepts either real conflict diagnostic while still requiring nonzero +status, exact winner retention, no published loser, and removal of the stage. +This is a corrected test assumption, not an implementation change. + +The complete fresh run at `moon-decoder-final-evidence` exited zero using the +verified Idriç `94dfd99bd3e376507fedc8611053b7173b2519f0` bundle and native +Grease executable SHA-256 +`7e31cd05b7a9d8fb2a4a9e003a7f3fcb0159138506d17f0fb28da8cbe22aa85c`. +The archive digest and every `payload.sha256` entry were independently checked. +`ASAN_OPTIONS=detect_leaks=0` was explicit for this host's orchestration; +this is not leak-analysis acceptance. Both the primary and corrected retained +index passed all 256 scalar checks; the existing alias, binary NUL/CRLF/C1, +BOM, process/source-integrity, exact real-capture and two compiled-mutant +controls passed. The supplemental checks passed as described above. + +The Idriç decoder and command source remain byte-identical to the prerequisite; +their digests and the independently rebuilt programs are in `moon-host.tsv`. +Chez program bytes depend on their build paths, so the independent program +digests do not purport to equal the earlier staging-root digests. Vocabulary +and full `git diff --check` passed with no fixture whitespace exception. + +These are Linux x86-64 host checks; DEX/ART, joined Prepaint/Arena admission, +Android delivery and physical-device evidence remain NOT_RUN. No new blocking +decoder capability was exposed. The next integration obligation remains a +decoded-representation consumer that preserves original byte identity and +independently establishes navigation/reader admission; this qualification does +not supply the missing redirect chain or implement that consumer. diff --git a/examples/autogenerated/source-byte-decoding/moon-host.tsv b/examples/autogenerated/source-byte-decoding/moon-host.tsv new file mode 100644 index 00000000..bbd07e5b --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/moon-host.tsv @@ -0,0 +1,31 @@ +schema ib-source-decoding-independent-host-v1 +implementation_commit f6c1c195f71070014e7aa5b254beea20051f91e5 +implementation_tree 364684f10d4cfa95f5996a6c6cdb68b21d686955 +scope HOST_IDRIC_BYTES_AND_NATIVE_GREASE +compiler_source 94dfd99bd3e376507fedc8611053b7173b2519f0 +compiler_archive_sha256 0be57deab9410e8d4c09c7198bf7e8ee0ec3e24121f76640147efa4a53d120da +grease_binary_sha256 7e31cd05b7a9d8fb2a4a9e003a7f3fcb0159138506d17f0fb28da8cbe22aa85c +decoder_module_sha256 e3b81a3450878f23a9c5228ca6695e64692bae4367560e5abdabe6adaed327f5 +decoder_command_source_sha256 bb81e9512d5c211af777d7e0b5db978a806ce951410724693b2494153608d9ab +independent_decoder_program_sha256 1ea35627128729e92f9361e15c5c605bb8aca6f93b77fc6e553ebf20f7afa204 +independent_semantic_program_sha256 f6bd2f5ee695d9359f60435b83b7db56cabdd5eb100d83a172acb5ab5552bb21 +normative_index_primary_url https://encoding.spec.whatwg.org/index-windows-1252.txt +normative_index_bytes 5667 +normative_index_sha256 2c5fa192d566f92b8b9e6e116255b1f94016b4ece5b9dacb3b98be70b8c60795 +retained_fixture_matches_primary BYTE_IDENTICAL +all_256_scalars PASS +provided_process_controls PASS +compiled_mutants_rejected PASS +original_navigation_YES PRESERVED +opaque_source_provenance PRESERVED +reader_admission NOT_GRANTED +duplicate_or_missing_descriptor_fields REFUSED_NO_DERIVATIVE +competing_destination_winner PRESERVED +losing_stage REMOVED +actual_google_body_bytes 93166 +actual_google_body_sha256 f71e991ab87271e77255fe2d2454cb4cb401fe47cdf953ea550c6e0fcd8afeb4 +actual_google_derivative BYTE_IDENTICAL +joined_Prepaint_Arena NOT_RUN +DEX_ART NOT_RUN +Android_delivery NOT_RUN +physical_device NOT_RUN diff --git a/examples/autogenerated/source-byte-decoding/moon-verify.grease b/examples/autogenerated/source-byte-decoding/moon-verify.grease new file mode 100644 index 00000000..0f45b844 --- /dev/null +++ b/examples/autogenerated/source-byte-decoding/moon-verify.grease @@ -0,0 +1,92 @@ +#!/usr/bin/env grease +# Independent source-decoder rebuild and publication/admission controls. +umask 077 +if (len(ARGV) !== 6) { + write -- 'usage: moon-verify.grease IDRIC GREASE GOOGLE_BODY GOOGLE_METADATA PRIMARY_INDEX NEW_EVIDENCE' >&2 + exit 2 +} +var compiler = ARGV[0] +var grease = ARGV[1] +var google = ARGV[2] +var google_metadata = ARGV[3] +var primary_index = ARGV[4] +var evidence = ARGV[5] +for path in (ARGV) { + if (not path.startsWith('/')) { write -- 'FAIL absolute input required' >&2; exit 2 } +} +if test -e $evidence { write -- 'FAIL refusing existing evidence' >&2; exit 2 } +var example = $(dirname -- $0) +var root = $(realpath -- "$example/../../..") +var source = "$evidence/source" +mkdir -p $source +cmp $primary_index "$root/tests/fixtures/index-windows-1252.txt" +cp -R "$root/src/." "$source/" +cp "$example/SourceDecodingChecks.idric" "$source/" +sha256sum "$root/src/IB/SourceDecoding.idric" "$root/src/SourceDecodingCommand.idric" $primary_index > "$evidence/inputs.sha256" +git -C $root rev-parse HEAD > "$evidence/source-ref.txt" +var previous = $(pwd) +cd $source +$compiler SourceDecodingCommand.idric -o source-decoder > "$evidence/command-compile.log" 2>&1 +$compiler SourceDecodingChecks.idric -o source-decoding-checks > "$evidence/checks-compile.log" 2>&1 +cd $previous +if rg -n '^[[:space:]]*(Error:|INTERNAL ERROR)' "$evidence/command-compile.log" "$evidence/checks-compile.log" { + write -- 'FAIL fresh compiler diagnostic' >&2 + exit 1 +} +var decoder = "$source/build/exec/source-decoder" +var checks = "$source/build/exec/source-decoding-checks" +sha256sum "${decoder}_app/source-decoder.so" "${checks}_app/source-decoding-checks.so" > "$evidence/programs.sha256" +$checks $primary_index "$evidence/all.raw" "$evidence/expected.utf8" > "$evidence/primary-index.log" +env GREASE=$grease $grease "$root/tests/source-decoding.grease" $checks $decoder $google $google_metadata > "$evidence/process.log" 2>&1 +env IDRIC=$compiler $grease "$root/tests/source-decoding-mutations.grease" > "$evidence/mutants.log" 2>&1 + +var raw = "$evidence/all.raw" +var descriptor = "$evidence/source.tsv" +var hash = $(sha256sum $raw | cut -d ' ' -f 1) +printf 'source_key\tmoon:source-generation\nsource_generation\t42-retained\nsha256\t%s\nbyte_count\t256\ncharset\tISO-8859-1\nnavigation_admitted\tYES\nredirect_count\t1\nopaque_origin\tretained provenance with spaces\n' $hash > $descriptor +$grease "$root/bin/ib_decode_html_bytes.grease" $decoder $descriptor $raw "$evidence/yes-derived" > "$evidence/yes-result.log" 2>&1 +cmp $descriptor "$evidence/yes-derived/source.tsv" +cmp "$evidence/expected.utf8" "$evidence/yes-derived/decoded.utf8" +grep -Fx b'original_navigation_admitted\tYES' "$evidence/yes-derived/decode.tsv" +grep -Fx b'reader_admission\tNOT_GRANTED' "$evidence/yes-derived/decode.tsv" +grep -Fx b'opaque_origin\tretained provenance with spaces' "$evidence/yes-derived/source.tsv" +printf '\033[32mPASS\033[0m source navigation YES and opaque provenance preserved without reader admission\n' + +cp $descriptor "$evidence/duplicate.tsv" +printf 'charset\tISO-8859-1\n' >> "$evidence/duplicate.tsv" +try { $grease "$root/bin/ib_decode_html_bytes.grease" $decoder "$evidence/duplicate.tsv" $raw "$evidence/duplicate-derived" > "$evidence/duplicate.log" 2>&1 } +if (not _error.code) { write -- 'FAIL duplicate descriptor accepted' >&2; exit 1 } +test ! -e "$evidence/duplicate-derived" +grep -F 'malformed or incomplete source descriptor' "$evidence/duplicate.log" +sed '/^source_generation\t/d' $descriptor > "$evidence/missing.tsv" +try { $grease "$root/bin/ib_decode_html_bytes.grease" $decoder "$evidence/missing.tsv" $raw "$evidence/missing-derived" > "$evidence/missing.log" 2>&1 } +if (not _error.code) { write -- 'FAIL missing source generation accepted' >&2; exit 1 } +test ! -e "$evidence/missing-derived" +grep -F 'malformed or incomplete source descriptor' "$evidence/missing.log" +printf '\033[32mPASS\033[0m malformed descriptor refusals have no derivative\n' + +# A deterministic process fixture creates the winner after initial precheck. +printf '#!%s\n' $grease > "$evidence/competing-producer" +cat >> "$evidence/competing-producer" <<'EOF' +var decoder = ENV.MOON_REAL_DECODER +$decoder @ARGV +var destination = ENV.MOON_COMPETING_DESTINATION +mkdir $destination +printf 'retained winner\n' > "$destination/winner" +EOF +chmod +x "$evidence/competing-producer" +try { + env MOON_REAL_DECODER=$decoder MOON_COMPETING_DESTINATION="$evidence/competing-derived" \ + $grease "$root/bin/ib_decode_html_bytes.grease" "$evidence/competing-producer" $descriptor $raw "$evidence/competing-derived" > "$evidence/competing.log" 2>&1 +} +if (not _error.code) { write -- 'FAIL competing winner was reported as our publication' >&2; exit 1 } +grep -E 'not replacing|derivative publication conflicted' "$evidence/competing.log" +test $(cat "$evidence/competing-derived/winner") = 'retained winner' +test ! -e "$evidence/competing-derived/source.tsv" +test ! -e "$evidence/competing-derived/decode.tsv" +var leftovers = $(find $evidence -maxdepth 1 -name '.ib-decode.*' -print) +if (leftovers !== '') { write -- 'FAIL discarded staging directory survived' >&2; exit 1 } +cmp $descriptor "$evidence/yes-derived/source.tsv" +cmp "$evidence/expected.utf8" "$evidence/yes-derived/decoded.utf8" +printf '\033[32mPASS\033[0m competing winner and previous derivative survive; losing stage removed\n' +printf 'fresh_Idric_build\tPASS\nprimary_index_all256\tPASS\nprovided_process_controls\tPASS\ncompiled_mutants_rejected\tPASS\noriginal_navigation_YES\tPRESERVED\nopaque_provenance\tPRESERVED\nreader_admission\tNOT_GRANTED\nmalformed_descriptor\tREFUSED\ncompeting_winner\tPRESERVED\nlosing_stage\tREMOVED\nART\tNOT_RUN\njoined_Prepaint\tNOT_RUN\nphysical_device\tNOT_RUN\n' > "$evidence/result.tsv" diff --git a/src/IB/SourceDecoding.idric b/src/IB/SourceDecoding.idric new file mode 100644 index 00000000..d8759c87 --- /dev/null +++ b/src/IB/SourceDecoding.idric @@ -0,0 +1,98 @@ +module IB.SourceDecoding + +import Data.Text +import Data.List + +-- Named raw boundary: these are source octets, never decoded characters. +public export +SourceOctet : Type +SourceOctet = Bits8 + +public export +record HtmlDecodedText where + constructor DecodedHtml + declared_label : Text + selected_encoding : Text + original_byte_count : Number + decoded_text : Text + +export +decoder_identity : Text +decoder_identity = "whatwg-windows-1252-e56d49d9176e9a4-v1" + +ascii_whitespace : Char → Bool +ascii_whitespace character = elem (ord character) [9, 10, 12, 13, 32] + +ascii_lower : Char → Char +ascii_lower character = + let scalar = ord character in + if scalar >= 65 && scalar <= 90 then chr (scalar + 32) else character + +normalized_label : Text → Text +normalized_label label = + pack (map ascii_lower + (reverse (dropWhile ascii_whitespace + (reverse (dropWhile ascii_whitespace (unpack label)))))) + +export +windows_1252_label : Text → Bool +windows_1252_label label = elem (normalized_label label) + ["ansi_x3.4-1968", "ascii", "cp1252", "cp819", "csisolatin1", + "ibm819", "iso-8859-1", "iso-ir-100", "iso8859-1", "iso88591", + "iso_8859-1", "iso_8859-1:1987", "l1", "latin1", "us-ascii", + "windows-1252", "x-cp1252"] + +-- WHATWG index-windows-1252.txt identifier e56d49d9176e9a4... (2024-09-18). +-- ASCII and 0xA0..0xFF retain their scalar values. The 0x80..0x9F range +-- includes both punctuation and the five deliberately retained C1 controls. +windows_1252_scalar : SourceOctet → Int +windows_1252_scalar octet = + let value : Int = cast octet in + case value of + 128 ⇒ 0x20AC + 129 ⇒ 0x0081 + 130 ⇒ 0x201A + 131 ⇒ 0x0192 + 132 ⇒ 0x201E + 133 ⇒ 0x2026 + 134 ⇒ 0x2020 + 135 ⇒ 0x2021 + 136 ⇒ 0x02C6 + 137 ⇒ 0x2030 + 138 ⇒ 0x0160 + 139 ⇒ 0x2039 + 140 ⇒ 0x0152 + 141 ⇒ 0x008D + 142 ⇒ 0x017D + 143 ⇒ 0x008F + 144 ⇒ 0x0090 + 145 ⇒ 0x2018 + 146 ⇒ 0x2019 + 147 ⇒ 0x201C + 148 ⇒ 0x201D + 149 ⇒ 0x2022 + 150 ⇒ 0x2013 + 151 ⇒ 0x2014 + 152 ⇒ 0x02DC + 153 ⇒ 0x2122 + 154 ⇒ 0x0161 + 155 ⇒ 0x203A + 156 ⇒ 0x0153 + 157 ⇒ 0x009D + 158 ⇒ 0x017E + 159 ⇒ 0x0178 + _ ⇒ value + +recognized_bom : List SourceOctet → Bool +recognized_bom (0xEF :: 0xBB :: 0xBF :: _) = True +recognized_bom (0xFE :: 0xFF :: _) = True +recognized_bom (0xFF :: 0xFE :: _) = True +recognized_bom _ = False + +export +decode_html_bytes : Text → List SourceOctet → Either Text HtmlDecodedText +decode_html_bytes label octets = + if not (windows_1252_label label) then Left "unsupported-html-encoding-label" + else if recognized_bom octets then Left "bom-selects-an-unimplemented-decoder" + else Right (DecodedHtml label "windows-1252" (length octets) + (pack (map (chr . windows_1252_scalar) octets))) diff --git a/src/SourceDecodingCommand.idric b/src/SourceDecodingCommand.idric new file mode 100644 index 00000000..2515d600 --- /dev/null +++ b/src/SourceDecodingCommand.idric @@ -0,0 +1,85 @@ +module SourceDecodingCommand + +import Data.Text +import Data.Buffer +import System +import System.File +import System.File.Buffer +import IB.SourceDecoding + +-- This command reads a bounded frozen byte file. It neither acquires a source +-- nor supplies canonical-store, redirect or navigation admission evidence. +source_byte_limit : Int +source_byte_limit = 4194304 + +covering +read_source_octets : Text → IO (Either Text (List SourceOctet)) +read_source_octets path = do + opened ← openFile path Read + case opened of + Left _ ⇒ pure (Left "source-open-failed") + Right file ⇒ do + sized ← fileSize file + result ← case sized of + Left _ ⇒ pure (Left "source-size-unavailable") + Right count ⇒ + if count < 0 || count > source_byte_limit + then pure (Left "source-outside-four-mebibyte-bound") + else do + allocated ← newBuffer count + case allocated of + Nothing ⇒ pure (Left "source-buffer-unavailable") + Just buffer ⇒ do + read_result ← readBufferData file buffer 0 count + case read_result of + Left _ ⇒ pure (Left "source-read-failed") + Right actual_count ⇒ + if not (actual_count ≟ count) then pure (Left "short-source-read") + else do + probe ← newBuffer 1 + case probe of + Nothing ⇒ pure (Left "source-probe-unavailable") + Just tail_buffer ⇒ do + tail_result ← readBufferData file tail_buffer 0 1 + case tail_result of + Right 0 ⇒ map Right (bufferData' buffer) + _ ⇒ pure (Left "source-size-changed-or-eof-unavailable") + closeFile file + pure result + +write_decoded_utf8 : Text → Text → IO (Either Text ()) +write_decoded_utf8 path text = do + let count = stringByteLength text + allocated ← newBuffer count + case allocated of + Nothing ⇒ pure (Left "decoded-buffer-unavailable") + Just buffer ⇒ do + setString buffer 0 text + written ← writeBufferToFile path buffer count + case written of + Left _ ⇒ pure (Left "decoded-write-failed") + Right () ⇒ pure (Right ()) + +covering +main : IO () +main = do + arguments ← getArgs + case arguments of + [_, label, input_path, output_path] ⇒ do + if input_path ≟ output_path then die "source-decoding: output-is-input" else pure () + read_result ← read_source_octets input_path + case read_result of + Left reason ⇒ die ("source-decoding: " ++ reason) + Right octets ⇒ + case decode_html_bytes label octets of + Left reason ⇒ die ("source-decoding: " ++ reason) + Right decoded ⇒ do + written ← write_decoded_utf8 output_path decoded.decoded_text + case written of + Left reason ⇒ die ("source-decoding: " ++ reason) + Right () ⇒ do + putStrLn ("original_byte_count\t" ++ show decoded.original_byte_count) + putStrLn ("decoded_codepoint_count\t" ++ show (length (unpack decoded.decoded_text))) + putStrLn ("selected_encoding\t" ++ decoded.selected_encoding) + putStrLn ("decoder_identity\t" ++ decoder_identity) + _ ⇒ die "usage: ib-source-decoder LABEL RAW_BYTES DISPOSABLE_UTF8" diff --git a/tests/fixtures/index-windows-1252.txt b/tests/fixtures/index-windows-1252.txt new file mode 100644 index 00000000..af1edbcd --- /dev/null +++ b/tests/fixtures/index-windows-1252.txt @@ -0,0 +1,134 @@ +# For details on index index-windows-1252.txt see the Encoding Standard +# https://encoding.spec.whatwg.org/ +# +# Identifier: e56d49d9176e9a412283cf29ac9bd613f5620462f2a080a84eceaf974cfa18b7 +# Date: 2024-09-18 + + 0 0x20AC € (EURO SIGN) + 1 0x0081  () + 2 0x201A ‚ (SINGLE LOW-9 QUOTATION MARK) + 3 0x0192 ƒ (LATIN SMALL LETTER F WITH HOOK) + 4 0x201E „ (DOUBLE LOW-9 QUOTATION MARK) + 5 0x2026 … (HORIZONTAL ELLIPSIS) + 6 0x2020 † (DAGGER) + 7 0x2021 ‡ (DOUBLE DAGGER) + 8 0x02C6 ˆ (MODIFIER LETTER CIRCUMFLEX ACCENT) + 9 0x2030 ‰ (PER MILLE SIGN) + 10 0x0160 Š (LATIN CAPITAL LETTER S WITH CARON) + 11 0x2039 ‹ (SINGLE LEFT-POINTING ANGLE QUOTATION MARK) + 12 0x0152 Œ (LATIN CAPITAL LIGATURE OE) + 13 0x008D  () + 14 0x017D Ž (LATIN CAPITAL LETTER Z WITH CARON) + 15 0x008F  () + 16 0x0090  () + 17 0x2018 ‘ (LEFT SINGLE QUOTATION MARK) + 18 0x2019 ’ (RIGHT SINGLE QUOTATION MARK) + 19 0x201C “ (LEFT DOUBLE QUOTATION MARK) + 20 0x201D ” (RIGHT DOUBLE QUOTATION MARK) + 21 0x2022 • (BULLET) + 22 0x2013 – (EN DASH) + 23 0x2014 — (EM DASH) + 24 0x02DC ˜ (SMALL TILDE) + 25 0x2122 ™ (TRADE MARK SIGN) + 26 0x0161 š (LATIN SMALL LETTER S WITH CARON) + 27 0x203A › (SINGLE RIGHT-POINTING ANGLE QUOTATION MARK) + 28 0x0153 œ (LATIN SMALL LIGATURE OE) + 29 0x009D  () + 30 0x017E ž (LATIN SMALL LETTER Z WITH CARON) + 31 0x0178 Ÿ (LATIN CAPITAL LETTER Y WITH DIAERESIS) + 32 0x00A0   (NO-BREAK SPACE) + 33 0x00A1 ¡ (INVERTED EXCLAMATION MARK) + 34 0x00A2 ¢ (CENT SIGN) + 35 0x00A3 £ (POUND SIGN) + 36 0x00A4 ¤ (CURRENCY SIGN) + 37 0x00A5 ¥ (YEN SIGN) + 38 0x00A6 ¦ (BROKEN BAR) + 39 0x00A7 § (SECTION SIGN) + 40 0x00A8 ¨ (DIAERESIS) + 41 0x00A9 © (COPYRIGHT SIGN) + 42 0x00AA ª (FEMININE ORDINAL INDICATOR) + 43 0x00AB « (LEFT-POINTING DOUBLE ANGLE QUOTATION MARK) + 44 0x00AC ¬ (NOT SIGN) + 45 0x00AD ­ (SOFT HYPHEN) + 46 0x00AE ® (REGISTERED SIGN) + 47 0x00AF ¯ (MACRON) + 48 0x00B0 ° (DEGREE SIGN) + 49 0x00B1 ± (PLUS-MINUS SIGN) + 50 0x00B2 ² (SUPERSCRIPT TWO) + 51 0x00B3 ³ (SUPERSCRIPT THREE) + 52 0x00B4 ´ (ACUTE ACCENT) + 53 0x00B5 µ (MICRO SIGN) + 54 0x00B6 ¶ (PILCROW SIGN) + 55 0x00B7 · (MIDDLE DOT) + 56 0x00B8 ¸ (CEDILLA) + 57 0x00B9 ¹ (SUPERSCRIPT ONE) + 58 0x00BA º (MASCULINE ORDINAL INDICATOR) + 59 0x00BB » (RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK) + 60 0x00BC ¼ (VULGAR FRACTION ONE QUARTER) + 61 0x00BD ½ (VULGAR FRACTION ONE HALF) + 62 0x00BE ¾ (VULGAR FRACTION THREE QUARTERS) + 63 0x00BF ¿ (INVERTED QUESTION MARK) + 64 0x00C0 À (LATIN CAPITAL LETTER A WITH GRAVE) + 65 0x00C1 Á (LATIN CAPITAL LETTER A WITH ACUTE) + 66 0x00C2  (LATIN CAPITAL LETTER A WITH CIRCUMFLEX) + 67 0x00C3 à (LATIN CAPITAL LETTER A WITH TILDE) + 68 0x00C4 Ä (LATIN CAPITAL LETTER A WITH DIAERESIS) + 69 0x00C5 Å (LATIN CAPITAL LETTER A WITH RING ABOVE) + 70 0x00C6 Æ (LATIN CAPITAL LETTER AE) + 71 0x00C7 Ç (LATIN CAPITAL LETTER C WITH CEDILLA) + 72 0x00C8 È (LATIN CAPITAL LETTER E WITH GRAVE) + 73 0x00C9 É (LATIN CAPITAL LETTER E WITH ACUTE) + 74 0x00CA Ê (LATIN CAPITAL LETTER E WITH CIRCUMFLEX) + 75 0x00CB Ë (LATIN CAPITAL LETTER E WITH DIAERESIS) + 76 0x00CC Ì (LATIN CAPITAL LETTER I WITH GRAVE) + 77 0x00CD Í (LATIN CAPITAL LETTER I WITH ACUTE) + 78 0x00CE Î (LATIN CAPITAL LETTER I WITH CIRCUMFLEX) + 79 0x00CF Ï (LATIN CAPITAL LETTER I WITH DIAERESIS) + 80 0x00D0 Ð (LATIN CAPITAL LETTER ETH) + 81 0x00D1 Ñ (LATIN CAPITAL LETTER N WITH TILDE) + 82 0x00D2 Ò (LATIN CAPITAL LETTER O WITH GRAVE) + 83 0x00D3 Ó (LATIN CAPITAL LETTER O WITH ACUTE) + 84 0x00D4 Ô (LATIN CAPITAL LETTER O WITH CIRCUMFLEX) + 85 0x00D5 Õ (LATIN CAPITAL LETTER O WITH TILDE) + 86 0x00D6 Ö (LATIN CAPITAL LETTER O WITH DIAERESIS) + 87 0x00D7 × (MULTIPLICATION SIGN) + 88 0x00D8 Ø (LATIN CAPITAL LETTER O WITH STROKE) + 89 0x00D9 Ù (LATIN CAPITAL LETTER U WITH GRAVE) + 90 0x00DA Ú (LATIN CAPITAL LETTER U WITH ACUTE) + 91 0x00DB Û (LATIN CAPITAL LETTER U WITH CIRCUMFLEX) + 92 0x00DC Ü (LATIN CAPITAL LETTER U WITH DIAERESIS) + 93 0x00DD Ý (LATIN CAPITAL LETTER Y WITH ACUTE) + 94 0x00DE Þ (LATIN CAPITAL LETTER THORN) + 95 0x00DF ß (LATIN SMALL LETTER SHARP S) + 96 0x00E0 à (LATIN SMALL LETTER A WITH GRAVE) + 97 0x00E1 á (LATIN SMALL LETTER A WITH ACUTE) + 98 0x00E2 â (LATIN SMALL LETTER A WITH CIRCUMFLEX) + 99 0x00E3 ã (LATIN SMALL LETTER A WITH TILDE) +100 0x00E4 ä (LATIN SMALL LETTER A WITH DIAERESIS) +101 0x00E5 å (LATIN SMALL LETTER A WITH RING ABOVE) +102 0x00E6 æ (LATIN SMALL LETTER AE) +103 0x00E7 ç (LATIN SMALL LETTER C WITH CEDILLA) +104 0x00E8 è (LATIN SMALL LETTER E WITH GRAVE) +105 0x00E9 é (LATIN SMALL LETTER E WITH ACUTE) +106 0x00EA ê (LATIN SMALL LETTER E WITH CIRCUMFLEX) +107 0x00EB ë (LATIN SMALL LETTER E WITH DIAERESIS) +108 0x00EC ì (LATIN SMALL LETTER I WITH GRAVE) +109 0x00ED í (LATIN SMALL LETTER I WITH ACUTE) +110 0x00EE î (LATIN SMALL LETTER I WITH CIRCUMFLEX) +111 0x00EF ï (LATIN SMALL LETTER I WITH DIAERESIS) +112 0x00F0 ð (LATIN SMALL LETTER ETH) +113 0x00F1 ñ (LATIN SMALL LETTER N WITH TILDE) +114 0x00F2 ò (LATIN SMALL LETTER O WITH GRAVE) +115 0x00F3 ó (LATIN SMALL LETTER O WITH ACUTE) +116 0x00F4 ô (LATIN SMALL LETTER O WITH CIRCUMFLEX) +117 0x00F5 õ (LATIN SMALL LETTER O WITH TILDE) +118 0x00F6 ö (LATIN SMALL LETTER O WITH DIAERESIS) +119 0x00F7 ÷ (DIVISION SIGN) +120 0x00F8 ø (LATIN SMALL LETTER O WITH STROKE) +121 0x00F9 ù (LATIN SMALL LETTER U WITH GRAVE) +122 0x00FA ú (LATIN SMALL LETTER U WITH ACUTE) +123 0x00FB û (LATIN SMALL LETTER U WITH CIRCUMFLEX) +124 0x00FC ü (LATIN SMALL LETTER U WITH DIAERESIS) +125 0x00FD ý (LATIN SMALL LETTER Y WITH ACUTE) +126 0x00FE þ (LATIN SMALL LETTER THORN) +127 0x00FF ÿ (LATIN SMALL LETTER Y WITH DIAERESIS) diff --git a/tests/source-decoding-mutations.grease b/tests/source-decoding-mutations.grease new file mode 100644 index 00000000..dddda40c --- /dev/null +++ b/tests/source-decoding-mutations.grease @@ -0,0 +1,40 @@ +#!/usr/bin/env grease +# Compile actual known-bad decoders; never edit the implementation under test. +umask 077 +var tests_directory = $(dirname -- $0) +var root = $(realpath -- "$tests_directory/..") +var compiler = ENV.IDRIC +var temporary = $(mktemp -d) +trap --add EXIT { rm -rf -- $temporary } + +proc compile_and_reject(source, name) { + var directory = "$temporary/$name" + mkdir -p -- "$directory/IB" + cp -- $source "$directory/IB/SourceDecoding.idric" + cp -- "$root/examples/autogenerated/source-byte-decoding/SourceDecodingChecks.idric" "$directory/SourceDecodingChecks.idric" + var previous = $(pwd) + cd $directory + $compiler SourceDecodingChecks.idric -o checks > "$directory/compile.log" 2>&1 + cd $previous + try { + "$directory/build/exec/checks" "$root/tests/fixtures/index-windows-1252.txt" "$directory/unused.raw" "$directory/unused.utf8" > "$directory/result.log" 2>&1 + } + if (not _error.code) { write -- "FAIL $name mutant passed normative oracle" >&2; exit 1 } + grep -F 'FAIL all 256 octets match normative index' "$directory/result.log" > /dev/null + write -- "PASS compiled $name mutant rejected by normative scalar oracle" +} + +# Wrong mechanism: interpret every source octet as the identically numbered +# Unicode scalar, ignoring HTML's windows-1252 punctuation mapping. +sed 's/chr \. windows_1252_scalar/chr . cast/' "$root/src/IB/SourceDecoding.idric" > "$temporary/latin1.idric" +if cmp -s "$root/src/IB/SourceDecoding.idric" "$temporary/latin1.idric" { exit 1 } +compile_and_reject "$temporary/latin1.idric" 'naive-latin1' + +# Wrong mechanism: a non-WHATWG undefined-byte policy replaces five C1 slots. +sed -e 's/129 ⇒ 0x0081/129 ⇒ 0xFFFD/' \ + -e 's/141 ⇒ 0x008D/141 ⇒ 0xFFFD/' \ + -e 's/143 ⇒ 0x008F/143 ⇒ 0xFFFD/' \ + -e 's/144 ⇒ 0x0090/144 ⇒ 0xFFFD/' \ + -e 's/157 ⇒ 0x009D/157 ⇒ 0xFFFD/' \ + "$root/src/IB/SourceDecoding.idric" > "$temporary/replacement.idric" +compile_and_reject "$temporary/replacement.idric" 'undefined-byte-replacement' diff --git a/tests/source-decoding.grease b/tests/source-decoding.grease new file mode 100644 index 00000000..128374d8 --- /dev/null +++ b/tests/source-decoding.grease @@ -0,0 +1,130 @@ +#!/usr/bin/env grease +# Actual Idriç binary decoding and source-bound derivative publication. +umask 077 +var tests_directory = $(dirname -- $0) +var root = $(realpath -- "$tests_directory/..") +var runner = ENV.GREASE +if (len(ARGV) !== 4) { + write -- 'usage: source-decoding.grease SEMANTIC_CHECKS DECODER GOOGLE_BODY GOOGLE_HTTP_METADATA' >&2 + exit 2 +} +var checks = ARGV[0] +var decoder = ARGV[1] +var google = ARGV[2] +var google_metadata = ARGV[3] +var temporary = $(mktemp -d) +trap --add EXIT { rm -rf -- $temporary } + +proc descriptor(raw, path, charset='ISO-8859-1') { + var hash = $(sha256sum -- $raw | cut -d ' ' -f 1) + var count = $(wc -c < $raw | tr -d '[:space:]') + printf 'source_key\tfixture:retained-byte-decoding\nsource_generation\tfixture-generation-1\nsha256\t%s\nbyte_count\t%s\ncharset\t%s\nnavigation_admitted\tNO\nredirect_count\t1\n' $hash $count $charset > $path +} + +proc publish(producer, metadata, raw, destination) { + $runner "$root/bin/ib_decode_html_bytes.grease" $producer $metadata $raw $destination +} + +proc refuse(producer, metadata, raw, destination, expected, error) { + try { publish $producer $metadata $raw $destination > /dev/null 2> $error } + if (not _error.code) { write -- 'FAIL expected derivative refusal' >&2; exit 1 } + grep -F -- $expected $error > /dev/null + test ! -e $destination +} + +$checks "$root/tests/fixtures/index-windows-1252.txt" "$temporary/all.raw" "$temporary/expected.utf8" +$decoder ' ISO-8859-1 ' "$temporary/all.raw" "$temporary/direct.utf8" > "$temporary/direct.tsv" +cmp -- "$temporary/expected.utf8" "$temporary/direct.utf8" +test $(wc -c < "$temporary/all.raw") -eq 256 +# Independent byte-level controls avoid relying only on the test's shared +# UTF-8 buffer writer: 128 ASCII bytes + 81 indexed bytes + 192 Latin bytes. +test $(wc -c < "$temporary/direct.utf8") -eq 401 +head -c 128 -- "$temporary/all.raw" > "$temporary/ascii.raw" +head -c 128 -- "$temporary/direct.utf8" > "$temporary/ascii.utf8" +cmp -- "$temporary/ascii.raw" "$temporary/ascii.utf8" +test $(od -An -v -tx1 -j 128 -N 5 "$temporary/direct.utf8" | tr -d ' \n') = 'e282acc281' +test $(od -An -v -tx1 -j 209 -N 2 "$temporary/direct.utf8" | tr -d ' \n') = 'c2a0' +test $(od -An -v -tx1 -j 355 -N 2 "$temporary/direct.utf8" | tr -d ' \n') = 'c3a9' +write -- 'PASS binary command preserves every indexed scalar including NUL and C1' +try { $decoder 'ISO-8859-1' "$temporary/all.raw" "$temporary/all.raw" > /dev/null 2> "$temporary/same-path-error" } +if (not _error.code) { write -- 'FAIL decoder overwrote its input' >&2; exit 1 } +grep -F 'output-is-input' "$temporary/same-path-error" > /dev/null +test $(wc -c < "$temporary/all.raw") -eq 256 + +descriptor "$temporary/all.raw" "$temporary/source.tsv" +publish $decoder "$temporary/source.tsv" "$temporary/all.raw" "$temporary/derived" > /dev/null +cmp -- "$temporary/source.tsv" "$temporary/derived/source.tsv" +cmp -- "$temporary/expected.utf8" "$temporary/derived/decoded.utf8" +grep -Fx b'reader_admission\tNOT_GRANTED' "$temporary/derived/decode.tsv" > /dev/null +grep -Fx b'original_navigation_admitted\tNO' "$temporary/derived/decode.tsv" > /dev/null +grep -Fx b'declared_charset\tISO-8859-1' "$temporary/derived/decode.tsv" > /dev/null +test ! -e "$temporary/derived/input.raw" +test ! -e "$temporary/derived/producer.tsv" +var complete = $(sha256sum "$temporary/derived/decode.tsv" | cut -d ' ' -f 1) +test $(cat "$temporary/derived/complete") = $complete +write -- 'PASS disposable derivative preserves original evidence and grants no reader admission' + +sed 's/^byte_count\t256$/byte_count\t255/' "$temporary/source.tsv" > "$temporary/bad-count.tsv" +refuse $decoder "$temporary/bad-count.tsv" "$temporary/all.raw" "$temporary/bad-count" 'source byte count mismatch' "$temporary/error" +awk -F b'\t' 'BEGIN {OFS="\t"} $1=="sha256" {$2=sprintf("%064d",0)} {print}' "$temporary/source.tsv" > "$temporary/bad-hash.tsv" +refuse $decoder "$temporary/bad-hash.tsv" "$temporary/all.raw" "$temporary/bad-hash" 'source digest mismatch' "$temporary/error" +descriptor "$temporary/all.raw" "$temporary/unknown.tsv" 'utf-8' +refuse $decoder "$temporary/unknown.tsv" "$temporary/all.raw" "$temporary/unknown" 'unsupported-html-encoding-label' "$temporary/error" +printf '\357\273\277x' > "$temporary/bom.raw" +descriptor "$temporary/bom.raw" "$temporary/bom.tsv" +refuse $decoder "$temporary/bom.tsv" "$temporary/bom.raw" "$temporary/bom" 'bom-selects-an-unimplemented-decoder' "$temporary/error" +write -- 'PASS wrong source claims, unknown labels and BOMs never publish a derivative' +truncate -s 4194305 "$temporary/too-large.raw" +descriptor "$temporary/too-large.raw" "$temporary/too-large.tsv" +refuse $decoder "$temporary/too-large.tsv" "$temporary/too-large.raw" "$temporary/too-large" 'file exceeds decoding bound' "$temporary/error" +touch "$temporary/empty.raw" +descriptor "$temporary/empty.raw" "$temporary/empty.tsv" +publish $decoder "$temporary/empty.tsv" "$temporary/empty.raw" "$temporary/empty" > /dev/null +test ! -s "$temporary/empty/decoded.utf8" +grep -Fx b'decoded_codepoint_count\t0' "$temporary/empty/decode.tsv" > /dev/null +grep -Fx b'reader_admission\tNOT_GRANTED' "$temporary/empty/decode.tsv" > /dev/null +write -- 'PASS size bound and empty representation preserve decoding/admission separation' + +# These fixture producers run under the same verified Grease entrypoint. +printf '#!%s\n' $runner > "$temporary/nonzero" +cat >> "$temporary/nonzero" <<'EOF' +printf 'partial' > $[ARGV[2]] +exit 19 +EOF +chmod +x "$temporary/nonzero" +refuse "$temporary/nonzero" "$temporary/source.tsv" "$temporary/all.raw" "$temporary/nonzero-result" 'Idriç decoding producer refused or failed' "$temporary/error" +printf '#!%s\n' $runner > "$temporary/change-metadata" +cat >> "$temporary/change-metadata" <<'EOF' +var producer = ENV.IB_REAL_SOURCE_DECODER +$producer @ARGV +var parent = $(dirname -- $[ARGV[2]]) +printf 'changed\tmetadata\n' >> "$parent/source.tsv" +EOF +chmod +x "$temporary/change-metadata" +try { + env IB_REAL_SOURCE_DECODER=$decoder $runner "$root/bin/ib_decode_html_bytes.grease" "$temporary/change-metadata" "$temporary/source.tsv" "$temporary/all.raw" "$temporary/changed-metadata" > /dev/null 2> "$temporary/error" +} +if (not _error.code) { write -- 'FAIL changed source descriptor accepted' >&2; exit 1 } +grep -F 'source descriptor changed during decoding' "$temporary/error" > /dev/null +test ! -e "$temporary/changed-metadata" +cmp -- "$temporary/source.tsv" "$temporary/derived/source.tsv" +write -- 'PASS partial nonzero producer and source-evidence mutation preserve last valid derivative' + +# The observation is the exact existing capture, not another canonical source. +var observed_hash = $(sha256sum -- $google | cut -d ' ' -f 1) +if (observed_hash !== 'f71e991ab87271e77255fe2d2454cb4cb401fe47cdf953ea550c6e0fcd8afeb4') { exit 1 } +test $(wc -c < $google) -eq 93166 +grep -Fx b'content_type\ttext/html; charset=ISO-8859-1' $google_metadata > /dev/null +grep -Fx b'redirect_count\t1' $google_metadata > /dev/null +cp -- $google_metadata "$temporary/google-before.tsv" +descriptor $google "$temporary/google-source.tsv" +publish $decoder "$temporary/google-source.tsv" $google "$temporary/google-derived" > /dev/null +cmp -- $google "$temporary/google-derived/decoded.utf8" +cmp -- $google_metadata "$temporary/google-before.tsv" +cmp -- "$temporary/google-source.tsv" "$temporary/google-derived/source.tsv" +grep -Fx b'charset\tISO-8859-1' "$temporary/google-derived/source.tsv" > /dev/null +grep -Fx b'redirect_count\t1' "$temporary/google-derived/source.tsv" > /dev/null +grep -Fx b'reader_admission\tNOT_GRANTED' "$temporary/google-derived/decode.tsv" > /dev/null +write -- 'PASS actual retained Google bytes round-trip without relabeling or redirect admission' + +write -- 'PASS native Grease source-decoding process controls; Android and Arena admission remain separate'