From 6f7eb6d1d6fa06740d8de2bff45deaa3501bfaf9 Mon Sep 17 00:00:00 2001 From: Glenn Fiedler Date: Fri, 4 Sep 2026 17:45:37 +1000 Subject: [PATCH] The int_relative domain, terminal failure, min <= max on int128, and the shared corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit STANDARD.md and conformance/ are now vendored here, verbatim from mas-bandwidth/serialize, with CI jobs that fail when either drifts. The suite runs every vector in the corpus through ReadStream: an accepted vector must decode to the stated value and consume the stated bits, a refused vector must be refused with its destination unwritten. int_relative reconstructs current in a long, in every tier, and refuses the read unless the result lies in the domain (0 to 2^31 - 1) and strictly exceeds previous. The absolute tier's 32 raw bits are read unsigned, so a top-bit-set value is refused rather than accepted as a negative sequence number, and no tier writes its destination before the checks pass. The write and measure sides assert previous in the domain. ReadStream carries a failure latch. The first refusal sets it and every later read returns false, consuming no bits and writing no destination — zero-bit reads included, which the past-end check alone cannot catch. Only reset() clears it, and isFailed() reports it. int128 takes min <= max on all three streams, as every other ranged operation does. A degenerate range costs zero bits: the writer emits nothing, the reader consumes nothing and takes the value from min, and the zero-bit path no longer reaches the 1-to-32-bit group primitive. make test-release runs the whole suite with assertions off, beside the checked run, proving every read-side refusal is a check rather than an assert. Both are CI gates. Version 1.1.0. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 46 + Makefile | 15 +- README.md | 46 +- STANDARD.md | 927 +++++++++++++++++++++ USAGE.md | 81 +- conformance/int128.txt | 17 + conformance/int_relative.txt | 121 +++ src/serialize/BitStream.java | 17 +- src/serialize/MeasureStream.java | 3 +- src/serialize/ReadStream.java | 216 +++-- src/serialize/SerializeUtil.java | 3 + src/serialize/WriteStream.java | 12 +- test/serialize/tests/AllTests.java | 23 +- test/serialize/tests/ConformanceTests.java | 287 +++++++ test/serialize/tests/Int128Tests.java | 21 + test/serialize/tests/IntRelativeTests.java | 74 +- test/serialize/tests/StreamTests.java | 100 +++ 17 files changed, 1863 insertions(+), 146 deletions(-) create mode 100644 STANDARD.md create mode 100644 conformance/int128.txt create mode 100644 conformance/int_relative.txt create mode 100644 test/serialize/tests/ConformanceTests.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05cfbda..0a58330 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,4 +12,50 @@ jobs: with: distribution: temurin java-version: 21 + # the checked shape: write-side contracts are asserts and run here - run: make test JDK_HOME="$JAVA_HOME" + # the release shape: the same suite with assertions off, proving every + # read-side refusal binds without them + - run: make test-release JDK_HOME="$JAVA_HOME" + + spec-sync: + name: STANDARD.md and the corpus match upstream + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # STANDARD.md here is a VERBATIM VENDORED COPY of the wire-format + # specification that lives in mas-bandwidth/serialize. This job fails if the + # two have diverged. + # + # Why vendor rather than link: an implementer wants the normative text + # beside the implementation, at the revision they are reading. Why check: + # a duplicated normative document is the same failure as a duplicated + # constant, except it drifts silently and the ports are what break. + # + # If this fails, the upstream spec changed. Read the diff, port whatever + # it implies, then copy the new file across in the same commit. + - name: Compare STANDARD.md against mas-bandwidth/serialize + run: | + curl -fsSL -o /tmp/upstream-STANDARD.md \ + https://raw.githubusercontent.com/mas-bandwidth/serialize/main/STANDARD.md + echo "upstream: mas-bandwidth/serialize@$(git ls-remote https://github.com/mas-bandwidth/serialize.git refs/heads/main | cut -f1)" + if ! diff -u /tmp/upstream-STANDARD.md STANDARD.md; then + echo "::error::STANDARD.md has diverged from mas-bandwidth/serialize" + exit 1 + fi + echo "STANDARD.md matches upstream" + + # conformance/ is the shared corpus, vendored the same way and for the + # same reason: the suite runs every vector in it, and a corpus that has + # drifted is a suite testing the wrong contract. The whole directory is + # compared, so a vector file added upstream fails here rather than going + # unrun. + - name: Compare conformance/ against mas-bandwidth/serialize + run: | + git clone --quiet --depth 1 https://github.com/mas-bandwidth/serialize.git /tmp/upstream-serialize + if ! diff -ru /tmp/upstream-serialize/conformance conformance; then + echo "::error::conformance/ has diverged from mas-bandwidth/serialize" + exit 1 + fi + echo "conformance/ matches upstream" diff --git a/Makefile b/Makefile index 5a5c13e..9cdad97 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,9 @@ TEST_SRC := $(wildcard test/serialize/tests/*.java) CLASSES := build/classes TEST_CLASSES := build/test-classes -.PHONY: all test clean +.PHONY: all test test-release clean -all: test +all: test test-release # library: Java 17 language level, built and run on the pinned JDK 21 $(CLASSES)/.stamp: $(SRC) @@ -25,10 +25,17 @@ $(TEST_CLASSES)/.stamp: $(TEST_SRC) $(CLASSES)/.stamp $(JAVAC) --release 17 -Xlint:all -Werror -cp $(CLASSES) -d $(TEST_CLASSES) $(TEST_SRC) @touch $@ -# the suite runs with assertions enabled: write-side contracts are asserts, -# mirroring the family's debug/release split +# the checked shape: assertions enabled, so the write-side contracts — which +# are asserts, mirroring the family's debug/release split — are exercised test: $(TEST_CLASSES)/.stamp $(JAVA) -ea -cp $(CLASSES):$(TEST_CLASSES) serialize.tests.AllTests +# the release shape: assertions disabled, the same suite. Every refusal +# STANDARD.md places on a reader is a check rather than an assert, and this +# run is what proves it — a refusal that only held under -ea would pass here +# as an accepted stream. +test-release: $(TEST_CLASSES)/.stamp + $(JAVA) -da -cp $(CLASSES):$(TEST_CLASSES) serialize.tests.AllTests --release + clean: rm -rf build diff --git a/README.md b/README.md index 3f39611..8864acd 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,12 @@ family, wire compatible with the [Elixir](https://github.com/mas-bandwidth/serialize.elixir) libraries — the same values produce the same bytes in every implementation, so a stream written by one reads in any other. -[STANDARD.md](https://github.com/mas-bandwidth/serialize/blob/main/STANDARD.md) -in the C++ reference is the authority on every byte. +[STANDARD.md](STANDARD.md) — a verbatim vendored copy of the +specification in +[mas-bandwidth/serialize](https://github.com/mas-bandwidth/serialize), +which CI checks for drift — is the authority on every byte. + +Version 1.1.0 (`SerializeUtil.VERSION`). ## The surface @@ -31,7 +35,7 @@ primitive-specialized holder cells (`IntRef`, `LongRef`, `BoolRef`, - **Ranged integers**: `serializeInt`, `serializeInt64`, `serializeInt128` — offset from min in exactly the bit length of the range, unsigned-domain arithmetic so ranges wider than 2^63/2^127 are - exact, zero bits for a degenerate range. + exact, zero bits for a degenerate `min == max` range on every width. - **Unsigned helpers and bool**: `serializeUint8` / `16` / `32` / `64`, `serializeUint128` (the `UInt128Value` pair), `serializeBool`. - **Floats**: `serializeFloat` and `serializeDouble`, bit transparent @@ -43,7 +47,8 @@ primitive-specialized holder cells (`IntRef`, `LongRef`, `BoolRef`, validated on read in every mode); `serializeWideString` (one 32-bit group per UTF-16 code unit, no alignment anywhere). - **The relative integer**: `serializeIntRelative` — the flag ladder for - strictly increasing uint32 sequences, one bit for a difference of 1. + strictly increasing sequences over the domain 0 to 2^31 - 1, one bit + for a difference of 1, every tier's reconstruction checked on read. - **Fixed point**: `serializeFixed` at 8/16/32/64-bit storage and `serializeFixed128` at 128-bit storage — Q formats, the raw scaled integer as an exact ranged offset, byte identical to `serializeInt64` @@ -96,22 +101,33 @@ language level (`javac --release 17`), built and tested on the pinned JDK 21. A plain Makefile drives everything — no Maven, no Gradle: ``` -make test # build the library and tests, run the suite with -ea +make # both shapes below +make test # the suite with assertions on, the checked shape +make test-release # the same suite with assertions off, the release shape ``` ## Testing `make test` runs the suite with assertions enabled (`-ea`): writer -contracts are `assert` statements, so the tested shape is the checked -shape, and a plain `java` invocation without `-ea` is the release shape -— asserts compile to nothing at runtime, matching the C++ library's -`serialize_assert` under `NDEBUG`. The suite pins the family's golden -vectors byte for byte — the golden wire message covering every operation -class, the discriminating compressed-float vectors (bit patterns, not -tolerances), the string and wide-string pins, every relative-integer -tier, and the fixed point shapes at every group count — plus a sabotage -sweep proving every consumed bit of the golden stream is load bearing, -refusal proofs for hostile input, and the measure bound. +contracts are `assert` statements, so this is the checked shape. +`make test-release` runs the same suite with assertions disabled — the +release shape, where asserts compile to nothing at runtime, matching the +C++ library's `serialize_assert` under `NDEBUG` — which is what proves +the read side's refusals are checks rather than asserts. Both are CI +gates. + +The suite runs every vector in [`conformance/`](conformance), the +family's shared corpus, vendored from mas-bandwidth/serialize and +checked for drift by CI: an accepted vector must decode to the stated +value and consume the stated bits, a refused vector must be refused, and +nothing regenerates its own expectations. It also pins the family's +golden vectors byte for byte — the golden wire message covering every +operation class, the discriminating compressed-float vectors (bit +patterns, not tolerances), the string and wide-string pins, every +relative-integer tier, and the fixed point shapes at every group count — +plus a sabotage sweep proving every consumed bit of the golden stream is +load bearing, refusal and terminality proofs for hostile input, and the +measure bound. Benchmarking for the serialize family lives in [mas-bandwidth/schema](https://github.com/mas-bandwidth/schema)'s data-driven bench, which measures the generated codecs across every language on one corpus. diff --git a/STANDARD.md b/STANDARD.md new file mode 100644 index 0000000..467275c --- /dev/null +++ b/STANDARD.md @@ -0,0 +1,927 @@ +# serialize + +This document specifies the **wire format** produced and consumed by the +serialize library, precisely enough to write an independent implementation that +interoperates byte-for-byte. + +It describes a format, not an implementation. Nothing here constrains how you +structure your code. + +## Format version + +**The format version is 1.1.** It names the wire, not a library release. Two +endpoints interoperate when they run releases carrying the same format version, +and a release states which format version it implements. + +The rulings that moved the format off 1.0, which was this document as first +written: + +* **2026-08-15.** A degenerate range costs zero bits on every storage width. + `wstring` transmits UTF-16 code units, with surrogate conversion at the + boundary on a 4-byte `wchar_t` platform. Readers refuse malformed `string` + and `wstring` payloads. +* **2026-08-23.** `compressed_float` clamps the quantized integer to + `max_integer_value`. +* **2026-09-04.** `int_relative` carries the non-negative int32 domain, and + every tier's reconstruction is refused outside it. + +## Architecture + +serialize is a **bit packer**. Values are written as variable numbers of bits +rather than whole bytes, so a boolean costs one bit and an integer known to lie +in `[0,7]` costs three. + +Reading and writing are expressed once, as a single templated function per +message type, instantiated against a write stream, a read stream, or a measure +stream. The measure stream computes a conservative bound on the size a message +would occupy, without producing bytes — its obligations are specified in "The +Measure Stream" below. Everything else in this document concerns the bytes on +the wire. + +## General Conventions + +All multi-byte quantities are **little-endian**. + +The stream is accumulated in a **64-bit scratch word**. Bits are packed +**least-significant-bit first**: the first value written occupies the lowest +bits of the first word. When the scratch fills, the 64-bit word is copied to +the buffer in host byte order on little-endian machines, and byte-swapped on +big-endian machines. The result is the same bytes on the wire everywhere. + +A value of `n` bits, written when the scratch already holds `s` bits, occupies +bits `[s, s+n)` of the current word. A value that would cross the 64-bit +boundary is split: the low `64-s` bits complete the current word, and the +remainder begins the next. + +**Flush.** After the final value, any partially filled scratch word is written +out. The stream therefore always occupies a whole number of 8-byte words in the +writer's buffer, but the meaningful length is the number of bytes actually +required, rounded up to a byte. + +**Bit index and alignment.** The bit index is the count of bits written so far. +The stream is *aligned* when the bit index is a multiple of 8. The number of +bits needed to reach alignment is `(8 - (bit_index % 8)) % 8`. + +**Writes assume trusted data — doctrine, ratified** *(adopted 2026-08-15 from +the schema enactment; the ruling verbatim: "this is an intentional design +choice of serialize. the write path is trusted, and it's your responsibility +as the user of serialize library to write correctly. asserts in languages that +support them in debug only.")*. Writer inputs are stated as **obligations, not +defined behaviors**: this document owes a conforming writer exact bytes and +owes a misbehaving writer nothing. Within that doctrine, misuse surfaces by +each implementation's own convention, and costlier contracts — the UTF-8 +well-formedness contract under `string` is the type case, an O(n) check no +release path should carry — assert in checked builds, everywhere. A **checked +build** is a build with assertions enabled. This document uses that one term +wherever a check depends on the build. The read side is untouched by the +doctrine: readers face untrusted data, and every refusal rule this document +states binds in every build mode. + +## Bit-Level Primitives + +### bits + + serialize_bits( stream, value, bits ) + +Writes the low `bits` bits of `value`, where `bits` is in `[1,64]`. + +* For `bits <= 32` this is a single group of that many bits, and the value must + be less than `2^bits`. +* For `bits > 32` the value is split: the **low 32 bits are written first as a + 32-bit group**, then the remaining `bits - 32` high bits as a second group. + +Fixed-width helpers are aliases for exactly this, and carry no range +information of their own: + +| helper | equivalent to | +|---|---| +| `serialize_uint8( value )` | `serialize_bits( value, 8 )` | +| `serialize_uint16( value )` | `serialize_bits( value, 16 )` | +| `serialize_uint32( value )` | `serialize_bits( value, 32 )` | +| `serialize_uint64( value )` | `serialize_bits( value, 64 )` — low 32 then high 32 | + +### bool + + serialize_bool( stream, value ) + +One bit: `1` for true, `0` for false. + +### uint128 + + serialize_uint128( stream, value ) + +A 128-bit unsigned integer, always 128 bits on the wire: the **low 64-bit half +first**, then the high half, each half written exactly as +`serialize_bits( half, 64 )` — that is, four 32-bit groups from least +significant upward. When the stream is byte aligned, the result is the 16 +bytes of the value in little-endian order. + +The operation is representation-independent. A native `unsigned __int128`, the +library's emulated two-lane type (`lo` then `hi`, both `uint64_t`), and two +explicit `serialize_uint64` calls (low half first) all produce **byte-identical +wire**. An implementation with no 128-bit type reproduces the format exactly +with two 64-bit operations. + +### align + + serialize_align( stream ) + +Pads with **zero bits** until the bit index is a multiple of 8. If the stream +is already aligned, **nothing is written**. + +Readers must verify that the padding bits are zero and fail the read if they +are not. This makes malformed streams detectable rather than silently accepted. + +## Integers + +### int (ranged) + + serialize_int( stream, value, min, max ) + +The defining operation of the format. The number of bits used is determined +entirely by the range: + + bits_required( min, max ) = ( min == max ) ? 0 : 32 - count_leading_zeros( max - min ) + +`value - min` is written in that many bits. Note the consequences: + +* a range of `[0,7]` costs 3 bits; +* a range of `[0,8]` costs 4 bits; +* a degenerate range where `min == max` costs **zero bits** — the value is + known from the range alone and nothing is written. + +`min <= max` is the legal relation, in every build mode and for every ranged +operation in this document: `int`, `int64`, `int128` and `fixed`. The +degenerate range is a field a conforming implementation must accept, not a +misuse, so a checked build must assert `min <= max` and never `min < max`. The +writer emits nothing, the reader consumes nothing and takes the value from +`min`, and a measure adds zero bits. + +`compressed_float` is not a ranged operation and is excluded. It takes bounds +but quantizes across them, and a zero `delta` has no quantization to define, so +it requires `min < max` and a checked build asserts that. + +Readers must check that the decoded value lies within `[min,max]` and fail +otherwise. + +The range must be identical on both sides. The format carries no +self-description: a stream is only interpretable by a reader that performs the +same sequence of operations with the same parameters. + +### int64 (ranged) + + serialize_int64( stream, value, min, max ) + +The 64-bit counterpart of the ranged integer, and the only ranged 64-bit +operation. `bits_required64( min, max )` bits are used. If that is 32 or fewer, +the value is written as a single group of that many bits; otherwise the low 32 +bits are written first, followed by the remaining `bits - 32` high bits. + +**Do not confuse this with `serialize_uint64`**, which is not ranged — it is +`serialize_bits( value, 64 )` and always costs a full 64 bits. The names are +similar and the encodings are not. + +### int128 (ranged) + + serialize_int128( stream, value, min, max ) + +The 128-bit counterpart, and the only ranged 128-bit operation. +`bits_required128( min, max )` bits are used, where `min` and `max` are +converted to the unsigned 128-bit domain first, so a range wider than `2^127` +is exact rather than overflowing. The offset `value - min` is computed in that +same unsigned domain and written in 32-bit groups from least significant +upward — the same splitting rule as `serialize_bits` and the wide fixed point +path: `bits <= 32` is a single group, otherwise full 32-bit groups from the +bottom with the final group carrying the remainder, up to four groups. + +Where the range fits 64 bits or fewer the bytes are **identical to +`serialize_int64( value, min, max )`** over the same bounds. A field may +therefore be widened from 64 to 128 bits without changing the wire, provided +the bounds do not change. + +The bounds are runtime values, exactly as for `serialize_int` and +`serialize_int64`. The bit count comes from the runtime `bits_required128`, +which is available on every platform — including compilers with no native +`__int128`, where the emulated pair supplies every operation it needs. + +**Do not confuse this with `serialize_uint128`**, which is not ranged — it is +two `serialize_uint64` calls and always costs a full 128 bits. + +Readers must check that the decoded offset is at most `max - min` in the +unsigned domain and fail otherwise — reject, never clamp. + +**A degenerate range where `min == max` is legal and costs zero bits, on the +128-bit width exactly as on the narrower ones.** `bits_required128( min, max )` +is zero, and the rule stated under `serialize_int` applies unchanged: `min <= +max` in every build mode, nothing on the wire, and the value taken from `min`. + +### fixed (Q format, ranged) + + serialize_fixed( stream, value, integer_bits, fraction_bits, min, max ) + +A fixed point value held in an integer storage type of exactly `integer_bits + +fraction_bits` bits, with the sign bit counting toward `integer_bits` (Q48.16 +in an `int64_t`, Q112.16 in an `__int128`). The stored integer is the real +value scaled by `2^fraction_bits`. `min` and `max` are bounds in **whole real +units**, and all four parameters are compile-time constants of the call site — +they are part of the format, exactly like a ranged integer's bounds. + +The encoding is an offset encoding over the **raw** (scaled) bounds: + + raw_min = min << fraction_bits + raw_max = max << fraction_bits + bits = bit length of ( raw_max - raw_min ) — bits_required, at whatever width the range needs + +`raw_value - raw_min` is written in `bits` bits, split into 32-bit groups from +least significant upward exactly as `serialize_bits` splits wide values: +`bits <= 32` is a single group, otherwise full 32-bit groups from the bottom +with the final group carrying the remainder. For storage of 64 bits or fewer +the bytes are **identical to `serialize_int64( raw_value, raw_min, raw_max )`** +— fixed point adds no new wire structure, only the compile-time scaling +convention — and with `fraction_bits = 0` the operation *is* a ranged integer. + +Readers must check that the decoded offset is at most `raw_max - raw_min` and +fail otherwise — reject, never clamp. + +**A degenerate range where `min == max` is legal and costs zero bits — on +every storage width** *(adopted 2026-08-15 from the schema enactment)*. The +wire carries nothing and the reader recovers the value from the range alone: +the raw value is `min << fraction_bits`, exactly the rule the ranged integers +have always stated. The storage width must not change this: a Q112.16 field +over a degenerate range costs zero bits, not `fraction_bits` zeros. *(Until +2026-08-15 the implementations behaved four different ways here — zero bits, +`fraction_bits` of zeros on the wide path only, a compile failure and a +panic — the divergence the ruling closes.)* + +Because fixed point values are integers underneath, the round trip is +**exact**: unlike `compressed_float` there is no quantization step, and the +same raw value produces the same bytes and reads back bit-for-bit identical on +every platform. + +**The one rounding rule** *(adopted 2026-08-15 from the schema enactment)*: +the wire itself never rounds — the round trip above is exact — but wherever a +value is quantized into a Q format or narrowed out of one, fixed point rounds +ties **half away from zero**, everywhere it rounds: `( raw + half ) >> drop` +for `raw >= 0`, `-( ( -raw + half ) >> drop )` for `raw < 0`, with +`half = 1 << ( drop - 1 )`. The hazard, named: the naive arithmetic shift +*floors*, so an implementation that applies `( raw + half ) >> drop` to a +negative raw rounds ties toward +infinity and diverges by exactly one raw +step, on exact ties of negative raws only. A rounding rule is not wire shape — +no protocol identifier can see the divergence — so a conformance vector must +pin a negative tie value that distinguishes the two rules. + +### int_relative + + serialize_int_relative( stream, previous, current ) + +Encodes an increasing sequence compactly, where `current > previous`. Let +`difference = current - previous`. The encoding is a ladder of one-bit flags, +each answering "does it fit in this tier?": + +| tier | flag sequence | payload | difference range | +|---|---|---|---| +| `one-bit` | `1` | — | exactly 1 | +| `bounded-3` | `0 1` | `serialize_int( d, 2, 6 )` — 3 bits | 2 – 6 | +| `bounded-5` | `0 0 1` | `serialize_int( d, 7, 23 )` — 5 bits | 7 – 23 | +| `bounded-9` | `0 0 0 1` | `serialize_int( d, 24, 280 )` — 9 bits | 24 – 280 | +| `bounded-13` | `0 0 0 0 1` | `serialize_int( d, 281, 4377 )` — 13 bits | 281 – 4377 | +| `bounded-17` | `0 0 0 0 0 1` | `serialize_int( d, 4378, 69914 )` — 17 bits | 4378 – 69914 | +| `absolute` | `0 0 0 0 0 0` | `current` as 32 raw bits | anything | + +The tier names are this document's, and the conformance vectors use them. The +five `bounded-*` tiers are named for their payload width. + +A difference of 1 — the common case for sequence numbers — costs a single bit. + +**The final tier transmits `current`, not the difference.** Every tier above it +encodes the difference, so this reads as an inconsistency and is not: at full +width the subtraction buys nothing, and sending the absolute value lets the +reader check `current > previous` directly. The absolute form carries no +ordering guarantee of its own, so the reader checks it under the reconstruction +rule below. + +**The semantics are pinned: no wrapping** *(adopted 2026-08-15 from the schema +enactment; the ruling verbatim: "no wrapping sequence numbers. meant for +positive only and up to maximum only.")*. `serialize_int_relative` is strictly +increasing — `current > previous`, the reader fails otherwise — and no wrap +semantics exist: a caller with a wrapping counter unwraps it before +serializing. Wrap-around is not an encoding this operation carries, not now +and not by future amendment. + +**The domain is the non-negative int32 range, `0` to `2^31 - 1` inclusive.** +Both `previous` and `current` lie in it. The domain is a property of the +operation, not of the caller's storage type: a 64-bit or unsigned `previous` of +`2^31` is caller error everywhere, exactly as a negative one is. `previous` is +the caller's own state and never arrives off the wire, so a `previous` outside +the domain is caller error, asserted in checked builds, and this document +defines no wire meaning for it. + +**Every tier's reconstruction must be checked.** The reader must reconstruct +`current` in a width that cannot wrap, then compare the result against the +domain and against `previous`, and must refuse the read unless `current` lies +in the domain and is strictly greater than `previous`. That binds in the +`one-bit` tier, in each of the five `bounded-*` tiers, and in the `absolute` +tier. + +**The `absolute` tier's 32 raw bits are unsigned.** The group must be read as +an unsigned 32-bit value, so a value with the top bit set is outside the domain +and the rule above refuses it. A reader that reads the group into a signed +sequence type first has already left the domain, and the two readings disagree +about the same byte sequence. + +The refusal outcome is the one Reader Obligations states for every operation. + +## Floating Point + +### float + + serialize_float( stream, value ) + +The 32 bits of the IEEE-754 single-precision representation, written as a +32-bit group. No conversion, no compression. + +### double + + serialize_double( stream, value ) + +The 64 bits of the IEEE-754 double-precision representation, written as one +64-bit group. + +**Bit transparency — both directions** *(ratified 2026-08-15 from the #56 +re-audit; every implementation already complies)*. `float` and `double` are +transparent in both directions. Every bit pattern is legal on the wire — NaNs +with any payload, signaling NaNs, infinities, negative zero, denormals — and +the reader reproduces the transmitted pattern exactly: it must not +canonicalize, quiet, flush, or refuse any of them. There is no reader +latitude here — a reader that rejects NaN, or canonicalizes payloads, does +not conform; the read returns exactly the bits read. A round trip through any +conforming writer/reader pair preserves all 32 (or 64) bits. Conformance +vectors for these operations must compare **bit patterns, not values**: NaN +compares unequal to itself, `-0.0 == 0.0`, and a tolerance comparison cannot +see a quieted signaling bit, so a value-space comparison here proves nothing. + +### compressed_float + + serialize_compressed_float( stream, value, min, max, res ) + +A float quantized to a resolution. Let `delta = max - min` and +`values = delta / res`, clamped to `[1, 4294967040]` (the largest float below +`2^32`). Then: + + max_integer_value = ceil( values ) + bits = bits_required( 0, max_integer_value ) + +The writer clamps `(value - min) / delta` to `[0,1]`, multiplies by +`max_integer_value`, adds `0.5`, takes the floor, **clamps the resulting +integer to `max_integer_value`**, and writes the result in `bits` bits. The reader divides by `max_integer_value`, multiplies by `delta`, +and adds `min`. + +**This arithmetic is `float32`, and the two roundings are part of the format.** +The product `normalized * max_integer_value` rounds to `float32` BEFORE `0.5` +is added, and that sum rounds to `float32` before the floor. Two roundings, not +one. Specifically, an implementation must not: + +- widen any step to `double` (or any wider type) before the floor, and +- contract the multiply and the add into a fused multiply-add, which rounds + once instead of twice. Languages that permit contraction must suppress it + here. In C and C++ a plain `float` local is **not** sufficient: it suppresses + only statement-local contraction (clang's default `-ffp-contract=on`), and + under `-ffp-contract=fast` — GCC's default at every optimization level — the + compiler fuses straight through it. The rounding must be pinned by an + optimization barrier on the stored product (the C++ implementation's + `SERIALIZE_FLOAT_FORCE_ROUND`: an empty asm with a register output operand on + GCC/clang, a `volatile` store where inline asm is unavailable) or by building + with `-ffp-contract=off`. In Go an explicit `float32()` conversion around the + product suffices — the spec forbids fusing across it. Rust does not fuse + unless `mul_add` is called explicitly. + +**The integer clamp is normative — added 2026-08-23 (schema#109; ruling: +Glenn, live).** Once `max_integer_value >= 2^23` the `float32` ulp at the top +of the range reaches 1, so the rounded sum can exceed `max_integer_value` +itself. Without the clamp, 2,109,734,656 step counts emit a top-of-range code +the reader's own `integerValue > max_integer_value` check rejects, and 128 +step counts emit a code one bit wider than the field — where implementations +historically diverged on the wire (the C++ implementation leaked the extra +bit into the stream; serialize.cs masked it to zero). The clamp closes both +classes, costs one comparison on a path already doing a floor, and changes no +byte for any declaration outside `[2^23, 2^24)`. Witnesses every +implementation must pin, writing `max`: `[0, 8388609]` at resolution `1` (the +reader-rejects class) and `[0, 16777215]` at resolution `1` (the +wire-divergence class). + +This is not pedantry; it changes the bytes. Over `[0, 10]` at resolution +`0.01`, the required arithmetic quantizes `0.005` to `1`, `0.025` to `3`, +`0.105` to `11` and `9.995` to `1000`; widening to `double` yields `0`, `2`, +`10` and `999`. A value landing exactly on a quantum — `2.5` here — agrees +under every variant, so a conformance vector built only from such values will +pass while the wire is wrong. Vectors must include values that land between +quanta. + +The between-quanta values above discriminate a **widened** writer; a **fused** +writer is a separate class with its own discriminating band. Where +`max_integer_value` is below `2^23` the product's ulp is well under the `0.5` +being added and fusion almost never moves the integer; once +`max_integer_value` reaches `[2^23, 2^24)` — the integer clamp's band — the +product's ulp reaches `1` and fusion moves the quantized integer on mass: +measured exhaustively over `[0, 16777215]` at resolution `1`, a fused writer +moves 4,194,304 inputs (every even `float` in the top binade), the first being +`2^23` itself. Conformance vectors must include a value from this band — +`8388608.0` over that declaration is the pinned witness — or a fused writer +passes every vector below the band while shipping divergent wire above it. + +**The reader's arithmetic is pinned the same way.** The decode — divide by +`max_integer_value`, multiply by `delta`, add `min` — is `float32` with every +step rounding: the quotient rounds, the product rounds BEFORE `min` is added, +and the sum rounds. An implementation must not widen any step to `double`, and +must not contract the multiply and the add into a fused multiply-add — fused, +the decode rounds once instead of twice, and whenever `min` is non-zero the +decoded value can land one ulp away from the conformant result. That never +changes the bytes being read, but it changes the value obtained from them: a +value decoded on a fusing platform and re-encoded produces different wire, +which breaks round-tripping between conforming implementations. The same +suppression techniques the writer paragraph lists apply here, and conformance +vectors over a non-zero-`min` range must pin decoded values **bit-exactly** — +a tolerance comparison cannot see a one-ulp divergence. + +Readers must reject an integer greater than `max_integer_value`. + +**Non-finite inputs are non-conforming.** *(Adopted 2026-08-15; the ruling +verbatim: "it's non-conforming. also, attempting to send NaN or INF or +anything else through compressed float is non-conforming and should assert +out on write too.")* A declaration whose `delta = max - min` — or whose +`values = delta / res` — is not finite in `float32` is non-conforming, and +this document defines no wire meaning for it. Writing a non-finite value +(NaN, `+Inf`, `-Inf`) through `compressed_float` is non-conforming. +Conforming writers assert in checked builds, per the family's writer-trusted +model. The read path is untouched: the poison is in the declaration or the +input value, never on the wire, so there is nothing for a reader to refuse. + +This is lossy by construction: a round trip returns the nearest representable +quantum, not the original value. + +### object + + serialize_object( stream, object ) + +Invokes the object's own serialize function inline. It contributes **no bytes +of its own** — it is composition, not an encoding. Whatever the nested object +writes appears at exactly this position in the stream, with no framing, length +prefix, or alignment inserted around it. + +## Bytes and Strings + +### bytes + + serialize_bytes( stream, data, count ) + +**Aligns first**, then writes `count` raw bytes. The alignment is part of the +format, not an optimization — a reader that does not align will desynchronize. + +`count` is not written. Both sides must already agree on it. + +**`count` may be zero** *(ratified 2026-08-15 from the #56 re-audit; every +implementation already complies)*. The alignment is performed regardless: a +zero-length `serialize_bytes` pads to the byte boundary and writes nothing +else, and the reader performs and verifies the same alignment. Skipping the +alignment when `count` is zero — a plausible "optimization" — desynchronizes +every field that follows, and a round-trip self-test cannot catch it, because +both halves skip the same align. This clause is load-bearing for `string`, +whose empty case reaches exactly this path. + +### string + + serialize_string( stream, string, buffer_size ) + +A null-terminated narrow string. + +1. The length, as `serialize_int( length, 0, buffer_size - 1 )`. The bit cost + therefore depends on `buffer_size`, which both sides must agree on. +2. The characters, as `serialize_bytes` — **which aligns**. + +The terminator is not transmitted; the reader appends it. + +Because `buffer_size` is an operand rather than a transmitted value, the same +string serialized against different buffer sizes produces different bytes. + +**`string` payloads are well-formed UTF-8 by contract** *(adopted 2026-08-15 +from the schema enactment, writer-trusted per the doctrine above)*. The wire +shape is unchanged; what the `string` spelling adds is a **contract**: the +payload is well-formed UTF-8, the writer's obligation. Writing malformed UTF-8 +is a writer contract violation, asserted in checked builds where the language +supports them, and the conformance vectors carry only valid UTF-8. An +application with genuinely arbitrary payloads uses `serialize_bytes`, which +remains exactly that. *(Until 2026-08-15 this paragraph also promised no +mandatory read-path validation; the ruling below supersedes that half. The +writer's obligation stands unchanged.)* + +**Readers must refuse malformed `string` payloads** *(adopted 2026-08-15)*. +Two refusal rules, binding in every build mode like every other read-side rule +in this document: + +* **Invalid UTF-8 fails the read.** The payload the contract above promises is + the payload the reader insists on: a stream carrying malformed UTF-8 was not + produced by a conforming writer, and the reader refuses it rather than + handing it to the application. +* **An interior NUL fails the read** — a zero byte anywhere among the `length` + transmitted bytes. A conforming writer derives `length` from `strlen`, so no + zero byte can reach the wire from conformance; a stream carrying one gives + the payload **two lengths** — the wire length, and the `strlen` every + consumer downstream will compute — and everything between them rides + invisibly past whichever side uses the other. Refusing the byte closes the + smuggling primitive. (NUL is well-formed UTF-8, which is why this is its own + rule.) + +These are refusal rules, and refusal rules are format: an implementation that +skips one accepts streams a conforming implementation refuses. *(Cost, +recorded with the ruling: the string operations are the convenience path, not +the hot path — strings are rare in serialized game traffic, and a design +chasing minimal bandwidth does not send strings at all — so the O(n) +validation prices into traffic that performance-sensitive designs do not +carry.)* + +### wstring + + serialize_wstring( stream, string, buffer_size ) + +A null-terminated wide string. `buffer_size` counts **wide characters, not +bytes**. + +1. The length, as `serialize_int( length, 0, buffer_size - 1 )`. +2. Each character as a **32-bit group**, in order. + +**No alignment is performed anywhere in this operation** — this is the one +place where the wide-string path deliberately differs from its narrow +counterpart, which aligns via `serialize_bytes`. An implementation that mirrors +the narrow string path here will produce the wrong bytes. + +Wide characters are transmitted as 32 bits regardless of the local `wchar_t` +width. A group above `0xFFFF` is not a UTF-16 code unit, and **the reader must +refuse it on every platform**, whatever the local `wchar_t` can hold. Refusal +does not depend on the platform, so the same byte sequence is refused +everywhere. + +**Each 32-bit group carries one UTF-16 code unit — not one code point — and +the payload is well-formed UTF-16 by contract** *(adopted 2026-08-15 from the +schema enactment, writer-trusted per the doctrine above)*. Surrogate **pairs** +are valid — full Unicode, an astral character is two groups; an **unpaired** +surrogate is a writer contract violation, asserted in checked builds where +the language supports it. 2-byte and 4-byte `wchar_t` platforms must produce +**identical bytes**: the 4-byte platform converts at the boundary — splits +astral code points into surrogate pairs on write, recombines on read — because +the platform-compatibility claim this section used to make was false for astral +text when each platform transmitted its own `wchar_t` units. Basic-plane text +is unaffected on every platform. + +**Readers must refuse malformed `wstring` payloads** *(adopted 2026-08-15, the +same ruling as `string`'s)*. The same two rules, in UTF-16 terms, binding in +every build mode: + +* **An unpaired surrogate fails the read**: a high surrogate + (`0xD800`–`0xDBFF`) not immediately followed by a low surrogate + (`0xDC00`–`0xDFFF`), a low surrogate not immediately preceded by a high + surrogate, or a high surrogate as the final transmitted group. Well-formed + surrogate **pairs** remain valid — they are how astral text travels. +* **An interior NUL fails the read** — a zero group among the `length` + transmitted groups — by the same two-lengths logic as `string`: a conforming + writer derives `length` from `wcslen`, so a zero group is impossible from + conformance, and a stream carrying one is carrying a payload with two + lengths. + +## Worked Example + +The library's golden test serializes a fixed message and asserts an exact +112-byte output. One field is a wide string in a `wchar_t[8]` buffer +containing three characters — `0x043C`, `0x0438`, `0x0440` — and it produces +this 13-byte run: + + 0xE3 0x21 0x00 0x00 0xC0 0x21 0x00 0x00 0x00 0x22 0x00 0x00 0x00 + +Decoding it against this document: + +* `buffer_size` is 8, so the length field is `serialize_int( length, 0, 7 )`, + which is `bits_required(0,7)` = **3 bits**. +* `0xE3` is `1110 0011`. Its low 3 bits are `011` = **3**, the length. +* No alignment follows. The first character begins immediately at bit 3. +* The remaining 5 bits of `0xE3` are `11100` = `0x1C`, which is the low 5 bits + of `0x043C`. The next byte `0x21` supplies `0x043C >> 5`. The character is + **`0x043C`**. +* Two further 32-bit groups follow, yielding `0x0438` and `0x0440`. + +Total: 3 + 3×32 = 99 bits = 13 bytes once the following align pads to the byte +boundary. This matches, and it is the cheapest way to confirm an independent +implementation is correct. + +The message then aligns and continues with four fixed point fields. The first +is `serialize_fixed( value, 8, 8, -100, +100 )` — Q8.8, so `raw_min` is +`-100 << 8 = -25600`, the raw range is `51200`, and the field costs 16 bits. +The next two bytes of the golden vector are `0xC0 0x60`, which is the offset +`0x60C0 = 24768`; adding `raw_min` gives a raw value of `-832`, which is +`-3.25` in Q8.8 — exactly the value the golden message stores. + +After another align the message ends with two wide fixed point fields that +make the multi-group split load-bearing: a Q112.16 field over ±2^57 whole +units (75 bits — two full 32-bit groups from the bottom, then the 11-bit +remainder on top), and a Q64.64 field over the full int64 unit range (128 +bits — four 32-bit groups). A decoder that assembles the groups in the wrong +order, or puts the remainder anywhere but the most significant position, +decodes the wrong values here. + +## Read-only and write-only forms + +Every operation above has `read_` and `write_` variants — `read_string`, +`write_bits`, and so on — for code paths that only ever read or only ever +write, rather than sharing one templated function. + +**They produce byte-identical output to their `serialize_` counterpart.** They +exist for convenience and to avoid a branch, not to encode anything +differently. This document therefore specifies each operation once, under its +`serialize_` name. + +## The Measure Stream + +Until 2026-08-15 this document disclaimed the measure stream in its third +paragraph and never mentioned it again — the same silence `compressed_float` +once enjoyed, and the implementations had quietly split under it: one measured +alignment exactly from a running bit index, four charged the worst case. This +section replaces the silence with the ruling *(2026-08-15, verbatim: "measure +must be large enough to serialize the message but doesn't need to be exact. it +is used only for yojimbo message serialization to see if there is enough room +in the packet to definitely serialize a message.")*. + +**A measure is a bound, not the packet size.** A measure must report a size +**sufficient to serialize the message at any starting bit position**. It need +not be exact, and cannot be: alignment cost depends on the bit position the +message is later written at, which a measure does not know — the same message +costs different bits at different offsets, so no single number is exact for +all of them *(the ruling: "the exact is not possible, since align is going to +be different in 1st and 2nd times serialize is called. it is bit position +dependent.")*. + +**The expected implementation charges the worst case: 7 bits per +alignment-performing operation** — `align`, `bytes`, `string` — and exact +width for everything else, which is every other operation: nothing else in +this document is position-dependent. + +**A measure refuses nothing at runtime.** The measure follows the write +path's misuse model *(the writes-trusted doctrine above)*: invalid parameters +or out-of-range values are the caller's contract violation, asserted in +checked builds where the language supports it, and a measure never refuses at +runtime in release. A measure sits on the trusted side of the boundary — +nothing it sees came off a network. + +**Exact-from-zero accounting is non-conforming** *(the ruling: "so if some +implementations of serialize measure in other languages are exact, they +probably should not be. make them conservative bounds like in C++, and the +standard should specify this is what is expected.")*. A measure that computes +alignment from a running bit index starting at zero reports the exact cost of +writing the message from an aligned start — and **under-counts every unaligned +start**. The worked example: `{ bits(8); align; bits(8) }` is 16 bits — 2 +bytes — written from an aligned start, where the align is a no-op; written +from bit offset 1, the align pads 7 bits and the message spans 23 bits — 3 +bytes of room needed. The exact-from-zero answer of 2 bytes is not sufficient +at every starting position, which is the one thing a measure is for. The +conservative answer — 8 + 7 + 8 = 23 bits — is sufficient everywhere. + +**What a measure is for, and what it is not** *(rationale, recorded)*. The +operation exists so a packet assembler can ask "does this message definitely +fit in the space remaining?" — the yojimbo fits-check — and a conservative +bound answers exactly that question. Comparing a measure to a write's +`bytes_processed` and expecting equality is a misuse: the bound is not the +packet size. An application that needs the true bit count of a message from a +known starting position has always had the escape hatch — write it to a +scratch stream and read the count off the write. And the operation is probably +vestigial — *"we won't support it with the rANS encoder for example"* — +specified here because all nine implementations ship it and had already begun +to disagree, not because it is expected to survive into entropy-coded +encodings. + +**Testable**: for every message in the conformance corpus, +`measure >= bits written`, at every starting bit position; and the worked +example discriminates — a conservative measure reports 23 bits for +`{ bits(8); align; bits(8) }` where an exact-from-zero measure reports 16. + +## Reader Obligations + +The operations above state what the bytes mean. This section states what a +reader must **do**. These streams arrive from the network; for a parser of +untrusted input, whatever this document leaves unspecified is the attack +surface. + +**Reading past the end must fail.** An operation that would consume more bits +than remain in the stream fails the read. It must not produce a partial value, +zero-fill the missing bits, or wrap. The failure is terminal under the rule +below. + +**A refused primitive read must leave its destination unwritten.** The rule is +per primitive read: when a read of a scalar fails, the caller's value must be +exactly what it was before the call. A reader that assigns and then checks +leaves the caller holding a value the stream never carried, and a caller that +trusts the destination over the return code proceeds on it. + +Two things the rule does not reach. A read into a caller-owned buffer, which is +`bytes`, `string` and `wstring`, leaves that buffer's contents **unspecified** +after a refusal, and no implementation restructures a copy path for it. A +composite read, which is `object` or any sequence of reads over an array, may +leave earlier members written, because it is a sequence of primitive reads and +each one carries the rule alone. + +**Failure is terminal.** Nothing after a failing operation has a defined +position, so nothing after it is interpretable, and it must be the stream that +enforces that rather than the caller's discipline. Two shapes satisfy it: + +* **By latch**, where a stream object survives a failure. The stream carries a + failure state, the first failed read sets it, and every later read on that + stream must fail, consuming no bits and writing no destination. Poisoning the + position past the end of the buffer, so the existing past-end check refuses + every later read, is an admitted implementation of the latch and the + recommended one: it costs the read path nothing. +* **By construction**, where the failing read returns no successor stream or + unwinds. An immutable stream whose failing read returns an error and no + stream, and a reader that throws, satisfy the rule as written, because + neither hands the caller a stream to continue on. + +A failure persists until the stream is **re-initialized**, which is the +operation that points a stream at a new buffer, or until the stream is +discarded. An implementation with no re-initialization discards. + +**Past-end memory is an implementation contract, not a format concern.** The +stream is exactly its stated length, and no operation's meaning ever depends +on memory beyond it. An implementation may still *load* — never interpret — +bytes past the end as an artifact of how it reads: the C++ and C +implementations both load 64-bit windows at byte granularity and therefore +require their caller to allocate at least 8 bytes beyond the data. That is +the accepted best practice, and Implementation Law's buffer contract holds +implementations to it: machinery that avoids the slack requirement at the +cost of per-operation work in the hot path is a slower correct option: +conforming on the wire, refused as an implementation choice by the speed rule. +The format turns on none of this. Conformance requires that +loaded-but-uninterpreted bytes can never influence a decoded value or an +accept/reject decision, and that an implementation state which +allocation contract its caller is under — a caller holding the wrong contract +is reading out of bounds, and that is a property of the implementation's +documentation, not of the wire. + +**Trailing bits: writers must write zero; readers must not look; tools may +judge.** *(Adopted 2026-08-15; the ruling verbatim: "Yes, I am OK with +writers must write zero, readers must ignore non-zero. And it's good for a +check, we want to check-- was this really written by serialize? and this is +another way to encode this in.")* After the final operation of a message, up +to 7 bits may remain in the final byte. Three rules, one per party: + +* Writers **must emit zero** in the unused bits of the final byte. Every + implementation already does this by construction — the flushed scratch + beyond the bit index is zero — and the behavior is now an obligation + rather than an observation. It makes the encoding canonical: among + conforming writers, one logical stream is exactly one byte sequence. +* Readers **must not reject** a stream for the contents of those bits. No + read operation examines them. The zero-check obligation applies exactly + where an operation actually reads padding: `serialize_align`, and the + alignment step inside `serialize_bytes` and `serialize_string`. +* Non-zero trailing bits are a **provenance signal, not a protocol error**: + a conformance or diagnostic check may treat them as evidence that the + stream was not produced by a conforming writer — another way to ask "was + this really written by serialize?". Such a check lives in tooling and + validators, never on the read path, and its verdict never changes what a + reader accepts. + +**Refusal rules are part of the format.** The per-operation obligations stated +above — decoded values within `[min,max]`, decoded offsets within range, +alignment padding zero, `wstring` groups at or below `0xFFFF` — are refusal +rules, not advice. An implementation that skips one accepts streams a +conforming implementation refuses, and two implementations that disagree about +refusal disagree about the format. Every refusal rule is +testable by a vector that a conforming reader must reject. + +## Implementation Law + +*(Adopted 2026-08-16, after a six-implementation audit found invented contracts +replicating port to port. These rules govern how implementations are built, not +just what bytes they emit — because the audit proved the bytes stay honest only +when the practice does.)* + +**The job.** When this library is ported to a language, the job is to find +**the fastest correct implementation in that language.** Correct is defined by +this standard; fastest is defined by measurement. Everything below serves that +sentence. + +**Sources.** An implementation derives from exactly two sources: this standard, +and — where the standard is silent — the C++ implementation +(`mas-bandwidth/serialize`), which breaks the tie under the authority rule in +Provenance. **Sibling ports are never sources.** Copying a sibling's behavior +because it is the nearest working example is how inventions travel disguised as +specification; every port-to-port inheritance in the audit was carrying one. If +the standard lacks the information needed to implement correctly and fast, +**the standard is too loose: tighten it here, upstream — never improvise in a +port.** + +**The check model.** The caller is responsible for well-formed writes. Where +the language has checked builds, write-side contract validation uses them and +nothing else; release builds perform **zero** write-side validation in C and +C++, and the minimum the language permits elsewhere. Readers perform +exactly the refusal obligations of this standard (see Reader Obligations) plus +buffer-end reporting — and nothing more. A check that neither this standard +mandates nor the language forces is an invented contract, whatever its +justification sounds like: "safer", "more defensive", and "best practice" are +the exact phrases the audit found attached to every invention. + +A write-side assertion inspects the caller's value as the caller passed it, +before any narrowing the operation performs on the way to the wire. An +assertion placed after the narrowing sees a value already truncated to the +operation's width, so it cannot report the out-of-range input it exists to +diagnose. + +**The buffer contract.** Reading whole words through the end of the buffer, +with the allocation aligned up so the final word load is legal, is accepted +best practice — the C++ implementation does this, and implementations +should. Machinery that avoids the slack requirement at the cost of +per-operation work in the hot path is a slower correct option, and is refused +by the speed rule below. + +**Speed is normative.** Among correct implementations of an operation, the +fastest correct option is the conforming one. A new approach a port invents is +welcome **provided it is the fastest correct option** — beating the C++ +implementation is a contribution, and it should then adopt it. The named error +is choosing a slower correct option and calling it good: a port that is slower +than the C++ implementation for any reason other than a documented language +necessity is defective, and the deviation and its necessity must be documented +where the divergence lives. Performance parity is part of conformance in +spirit: C and C++ at total parity; systems languages within a few percent, +with every residual attributed to a named language mechanism. + +## Compatibility Notes + +* **The format is not self-describing.** There are no tags, lengths, or type + markers beyond what the operations imply. A stream is meaningless without the + exact sequence of calls that produced it. This is the source of its + compactness and the reason both endpoints must ship compatible code. +* **Ranges are part of the format.** Changing a `min`/`max` on one side changes + the bit width and silently desynchronizes everything after it. Range changes + are breaking changes. +* **Alignment is part of the format.** `serialize_bytes` and `serialize_string` + align; `serialize_bits`, `serialize_int`, `serialize_fixed`, + `serialize_uint128` and `serialize_wstring` do not. +* **Zero-bit fields are legal.** `min == max` writes nothing at all. + +## Provenance + +Written 2026-07-21 by Rowan, by reading the then-existing implementation and +verifying every claim against its golden test vector. + +**This document is the authority. Where this document and any implementation +disagree, the implementation is a bug.** Where this document is silent, the +behavior of the C++ implementation (`mas-bandwidth/serialize`) breaks the tie, +and it holds that standing only until this document is amended to state the +rule itself. A port never copies an implementation over the text of this +document. `serialize.h` is one implementation among nine: C, C++, C#, Dart, +Elixir, Go, Java, JavaScript and Rust. It was the first, which is a fact about +history and not about standing. + +**A rule this document states binds every implementation, including the ones +that do not have it yet.** This document leads. Where an implementation lags a +rule, its repository names the gap and the release that closes it, and the gap +is a defect in that implementation rather than a reading of this text. + +Until 2026-08-14 this section said the opposite, and the cost of that sentence +was measurable. Under it, five implementations quietly disagreed about +`compressed_float`'s precision and each was, by this document's own terms, +correct: C quantized in `double`, C++ and Go contracted the multiply and add +into a single fused multiply-add on arm64, C# and Rust rounded twice in +`float32`. Four different byte streams from one paragraph, and no divergence +was formally a defect, because whatever an implementation did was by definition +the format. + +Every normative statement here is testable, by a pinned vector or by an +explicit refusal test. An implementation conforms when it reproduces every +vector byte for byte and refuses everything this document says must be refused. + +**The shared corpus is the conformance instrument.** It is the `conformance/` +directory of this repository, one file per operation, holding the accepted and +refused vectors this document's rules require. Every implementation vendors and +syncs that directory the way it vendors this document, and its test suite must +run every vector in it. No checker reimplements the codec and then checks that +reimplementation against itself. A suite that regenerates its own expectations +proves only that a port agrees with itself, which is how one wrong reading of +this document travels to nine implementations under green results. + +**The vector format.** A vector file is text. `#` begins a comment, blank lines +separate records, and each record is `key` and value, one per line: + +| key | meaning | +|---|---| +| `operation` | the operation under test, once per record | +| `name` | a stable identifier for the vector | +| `param` | one parameter as `name = value`, repeated once per parameter | +| `bytes` | the stream, as hexadecimal byte pairs, empty for a zero-bit read | +| `expect` | the word `refused`, or `value = ` and the decoded value | +| `consumed` | bits a conforming reader consumes, accepted reads only | + +`consumed` is stated only for accepted reads. **After a refusal the stream +position is not part of the contract**, so no vector states it and no +implementation is judged on it. + +**Conformance vectors must discriminate.** A value taken from the middle of a +range, or one that lands where every plausible reading agrees, proves nothing — +the `compressed_float` divergence above survived years of green test suites +because every pinned value in every implementation landed exactly on a quantum, +where `float32`, `double` and a fused multiply-add all produce the same answer. +A vector that cannot fail is not evidence. diff --git a/USAGE.md b/USAGE.md index ebb7bd0..f1f8b64 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1,9 +1,11 @@ # Using serialize.java Everything the library does, by example. The wire format itself is -defined by the C++ reference's -[STANDARD.md](https://github.com/mas-bandwidth/serialize/blob/main/STANDARD.md); -this document teaches the Java surface that speaks it. +defined by [STANDARD.md](STANDARD.md) — a verbatim vendored copy of the +specification in +[mas-bandwidth/serialize](https://github.com/mas-bandwidth/serialize), +which CI checks for drift; this document teaches the Java surface that +speaks it. ```java import serialize.*; @@ -82,16 +84,33 @@ everything, because the wire is a trust boundary. On the read side, every failure — a truncated read, a value outside its range, nonzero alignment padding, a malformed string — returns `false`, -and hostile bytes never throw. A failed read is terminal for the stream: -nothing after the failing operation has a defined position. `reset(...)` -points the stream at data again and clears all state. +and hostile bytes never throw. + +**A failed read is terminal.** Nothing after the failing operation has a +defined position, so `ReadStream` latches: the first refusal sets the +failure flag, and every later read on that stream returns `false` +without consuming a bit or writing a destination — a zero-bit read +included. `isFailed()` reports the latch; `reset(...)` points the stream +at data again and clears it, and nothing else does. + +**A refused read leaves its destination unwritten.** When a read of a +scalar fails, the holder cell holds exactly what it held before the +call, so a caller that trusts the cell over the return code cannot +proceed on a value the stream never carried. Two limits: a read into a +caller-owned buffer — `serializeBytes`, `serializeString`, +`serializeWideString` — leaves that buffer's **contents unspecified** +after a refusal, and a composite read may leave earlier members written, +because it is a sequence of primitive reads and each one carries the +rule alone. ```java byte[] data = new byte[16]; // 1 byte of data + the 8-byte slack ReadStream r = new ReadStream( data, 1 ); // 8 bits of data r.serializeBits( v, 32 ); // -> false: past the end +r.serializeBits( v, 8 ); // -> false: the stream has failed, v untouched +r.isFailed(); // -> true -r.reset( data, 1 ); // point at data again, state cleared +r.reset( data, 1 ); // point at data again, the latch cleared r.serializeBits( v, 8 ); // -> true // an offset smuggled into the bit headroom of a range is refused @@ -118,7 +137,10 @@ asserts: refuse), never memory unsafety — the JVM's own bounds checks backstop the trusted path. -The wire for conforming writes is byte identical in both modes. +The wire for conforming writes is byte identical in both modes, and so +is every refusal: the read side's obligations are checks, never asserts. +`make test-release` runs the whole suite in the release shape to prove +it. ## Raw bits @@ -158,8 +180,10 @@ above `max - min` fails the read — reject, never clamp). computed in unsigned arithmetic so ranges wider than 2^63 are exact. `serializeInt128` extends it to 128 bits on the `Int128Value` pair, written in 32-bit groups least significant first; where the range fits -64 bits the bytes are identical to `serializeInt64`. Its bounds must -satisfy `min < max` strictly. +64 bits the bytes are identical to `serializeInt64`. `min <= max` is the +legal relation on every ranged width: a degenerate `min == max` costs +zero bits at 128 bits exactly as it does at 32, with nothing on the +wire, nothing consumed, and the value taken from `min`. ```java w.serializeInt64( new LongRef( -5000000000L ), -5000000000L, 5000000000L ); // 34 bits @@ -310,11 +334,19 @@ interior NUL groups, and unpaired, misordered or dangling surrogates. ## The relative integer `serializeIntRelative(previous, ref)` prices strictly increasing -unsigned 32-bit sequences — sequence numbers, ack chains. -`current > previous` always, no wrapping. A difference of 1 costs a -single bit; small differences ride payload tiers of 5/8/13/18/23 bits; -past the ladder, six zero flags carry `current` itself as 32 raw bits, -and the reader enforces the ordering on that absolute form too. +sequences — sequence numbers, ack chains. **The domain is `0` to +`2^31 - 1` inclusive**, and both `previous` and `current` lie in it. +`current > previous` always, no wrapping: a caller with a wrapping +counter unwraps it before serializing. A difference of 1 costs a single +bit; small differences ride payload tiers of 5/8/13/18/23 bits; past the +ladder, six zero flags carry `current` itself as 32 raw bits. + +The reader reconstructs `current` in a width that cannot wrap, in every +tier, and refuses the read unless the result lies in the domain and +strictly exceeds `previous` — the absolute tier's 32 raw bits are +unsigned, so a value with the top bit set is outside the domain and is +refused. A refused read is terminal and leaves the holder cell +untouched. ```java w.serializeIntRelative( 100, new IntRef( 101 ) ); // 1 bit @@ -323,8 +355,9 @@ w.serializeIntRelative( 100, new IntRef( 2100 ) ); // a mid-ladder tier r.serializeIntRelative( 100, seq ); // seq.value == 101 ``` -`previous` is caller state, not wire: both sides already know it. -Writing `current <= previous` is a contract violation, asserted under +`previous` is caller state, not wire: both sides already know it, and it +never arrives off the wire. Writing `current <= previous`, or a +`previous` outside the domain, is a contract violation, asserted under `-ea`. ## Fixed point @@ -404,12 +437,14 @@ of slack past the data. ## Wire compatibility The same values produce the same bytes in every family implementation. -This is not aspiration but pinned fact: the test suite carries the -family's golden vectors — including the golden wire message covering -every operation class, byte for byte — plus the discriminating float -vectors, the string and wide-string pins, every relative-integer tier, -and the fixed point shapes at every group count, all minted from the -canonical C++ reference's own output. If your message serializes with +This is not aspiration but pinned fact: the test suite runs every vector +in [`conformance/`](conformance) — the family's shared corpus, vendored +from mas-bandwidth/serialize and checked for drift by CI — and carries +the family's golden vectors: the golden wire message covering every +operation class byte for byte, the discriminating float vectors, the +string and wide-string pins, every relative-integer tier, and the fixed +point shapes at every group count, all minted from the canonical C++ +reference's own output. If your message serializes with the same declarations on both ends, a stream written by any family implementation reads in any other. diff --git a/conformance/int128.txt b/conformance/int128.txt new file mode 100644 index 0000000..16c92c5 --- /dev/null +++ b/conformance/int128.txt @@ -0,0 +1,17 @@ +# serialize conformance vectors: serialize_int128 +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# A degenerate range is legal and costs zero bits: STANDARD.md, "int (ranged)" +# for the rule and "int128 (ranged)" for the width. The stream is empty, the +# reader consumes nothing, and the value comes from min. A reader that requires +# min < max, or that consumes bits here, fails this vector. + +operation int128 +name degenerate-range-zero-bits +param min = 1267650600228229401496703205383 +param max = 1267650600228229401496703205383 +bytes +expect value = 1267650600228229401496703205383 +consumed 0 diff --git a/conformance/int_relative.txt b/conformance/int_relative.txt new file mode 100644 index 0000000..8b70cac --- /dev/null +++ b/conformance/int_relative.txt @@ -0,0 +1,121 @@ +# serialize conformance vectors: serialize_int_relative +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. `consumed` appears on accepted reads only, because +# after a refusal the stream position is not part of the contract. +# +# Every refusal here is the domain rule: STANDARD.md, "int_relative", where the +# domain is 0 to 2^31 - 1 inclusive and reconstruction is checked in every +# tier. Each refusal has an accept twin one step inside the domain, sharing its +# bytes, so a reader that refuses on the bytes rather than on the reconstructed +# value fails the twin. + +operation int_relative +name one-bit-refuse-past-domain +param previous = 2147483647 +bytes 01 +expect refused + +operation int_relative +name one-bit-accept-at-domain-top +param previous = 2147483646 +bytes 01 +expect value = 2147483647 +consumed 1 + +operation int_relative +name bounded-3-refuse-past-domain +param previous = 2147483647 +bytes 02 +expect refused + +operation int_relative +name bounded-3-accept-at-domain-top +param previous = 2147483645 +bytes 02 +expect value = 2147483647 +consumed 5 + +operation int_relative +name bounded-5-refuse-past-domain +param previous = 2147483647 +bytes 04 +expect refused + +operation int_relative +name bounded-5-accept-at-domain-top +param previous = 2147483640 +bytes 04 +expect value = 2147483647 +consumed 8 + +operation int_relative +name bounded-9-refuse-past-domain +param previous = 2147483647 +bytes 08 00 +expect refused + +operation int_relative +name bounded-9-accept-at-domain-top +param previous = 2147483623 +bytes 08 00 +expect value = 2147483647 +consumed 13 + +operation int_relative +name bounded-13-refuse-past-domain +param previous = 2147483647 +bytes 10 00 00 +expect refused + +operation int_relative +name bounded-13-accept-at-domain-top +param previous = 2147483366 +bytes 10 00 00 +expect value = 2147483647 +consumed 18 + +operation int_relative +name bounded-17-refuse-past-domain +param previous = 2147483647 +bytes 20 00 00 +expect refused + +operation int_relative +name bounded-17-accept-at-domain-top +param previous = 2147479269 +bytes 20 00 00 +expect value = 2147483647 +consumed 23 + +operation int_relative +name absolute-refuse-not-increasing +param previous = 2147483647 +bytes C0 FF FF FF 1F +expect refused + +operation int_relative +name absolute-accept-one-above-previous +param previous = 2147483646 +bytes C0 FF FF FF 1F +expect value = 2147483647 +consumed 38 + +operation int_relative +name absolute-refuse-top-bit-set +param previous = 100 +bytes 00 00 00 00 20 +expect refused + +operation int_relative +name absolute-refuse-all-bits-set +param previous = 100 +bytes C0 FF FF FF 3F +expect refused + +operation int_relative +name absolute-accept-domain-maximum +param previous = 100 +bytes C0 FF FF FF 1F +expect value = 2147483647 +consumed 38 diff --git a/src/serialize/BitStream.java b/src/serialize/BitStream.java index 6e8424a..febaa2b 100644 --- a/src/serialize/BitStream.java +++ b/src/serialize/BitStream.java @@ -9,7 +9,9 @@ * Every method returns false on refusal. On the write and measure sides the * data is trusted and misuse is caught by debug asserts only (run with -ea); * on the read side every refusal rule of STANDARD.md binds in every mode and - * hostile bytes never throw. + * hostile bytes never throw. A refused read leaves its scalar destination + * unwritten and is terminal for the stream — the exception is a caller-owned + * buffer, whose contents are unspecified after a refusal. */ public interface BitStream { @@ -31,7 +33,11 @@ public interface BitStream /** A ranged 64-bit integer: value - min in bitsRequired64(min,max) bits, low 32-bit group first past 32 bits. */ boolean serializeInt64( LongRef value, long min, long max ); - /** A ranged 128-bit integer: the offset in 32-bit groups from least significant upward. min must be strictly less than max. */ + /** + * A ranged 128-bit integer: the offset in 32-bit groups from least + * significant upward. min <= max, and a degenerate min == max range costs + * zero bits — nothing on the wire, the value taken from min. + */ boolean serializeInt128( Ref value, Int128Value min, Int128Value max ); /** An unsigned 8-bit integer: 8 raw bits. */ @@ -77,7 +83,12 @@ public interface BitStream */ boolean serializeWideString( Ref value, int bufferSize ); - /** The strictly increasing relative-integer ladder. current must exceed previous. */ + /** + * The strictly increasing relative-integer ladder over the domain 0 to + * 2^31 - 1 inclusive. current must exceed previous, and both lie in the + * domain; a read whose reconstruction leaves the domain or fails to exceed + * previous is refused. + */ boolean serializeIntRelative( int previous, IntRef current ); /** diff --git a/src/serialize/MeasureStream.java b/src/serialize/MeasureStream.java index 4a67022..9f455ff 100644 --- a/src/serialize/MeasureStream.java +++ b/src/serialize/MeasureStream.java @@ -104,7 +104,7 @@ public boolean serializeInt64( LongRef value, long min, long max ) @Override public boolean serializeInt128( Ref value, Int128Value min, Int128Value max ) { - assert min.compareTo( max ) < 0; + assert min.compareTo( max ) <= 0; assert value.value.compareTo( min ) >= 0; assert value.value.compareTo( max ) <= 0; bitsWritten += SerializeUtil.bitsRequired128( min.toUnsigned(), max.toUnsigned() ); @@ -211,6 +211,7 @@ public boolean serializeWideString( Ref value, int bufferSize ) @Override public boolean serializeIntRelative( int previous, IntRef current ) { + assert previous >= 0; // the domain: 0 to 2^31 - 1, previous and current alike assert previous < current.value; int difference = current.value - previous; diff --git a/src/serialize/ReadStream.java b/src/serialize/ReadStream.java index f6665b5..d454c21 100644 --- a/src/serialize/ReadStream.java +++ b/src/serialize/ReadStream.java @@ -8,13 +8,28 @@ * * The read side faces untrusted data: every refusal rule of STANDARD.md * binds here in every build mode. Out-of-range or truncated input returns - * false; hostile bytes never throw. A failed read is terminal for the - * stream — nothing after the failing operation has a defined position. + * false; hostile bytes never throw. + * + * A refused read leaves its scalar destination unwritten — the holder cell + * holds what it held before the call — except for a caller-owned buffer, + * whose contents are unspecified after a refusal. + * + * A failed read is terminal: nothing after the failing operation has a + * defined position, so the stream latches. Every read after the first + * refusal returns false, consuming no bits and writing no destination, until + * {@link #reset} points the stream at data again. */ public final class ReadStream implements BitStream { private final BitReader reader; + /** + * The failure latch. The first refused read sets it, every later read on + * this stream refuses without consuming a bit or writing a destination, + * and only {@link #reset} clears it. + */ + private boolean failed; + /** * @param buffer the buffer to read from. The array must extend at least * 8 bytes past {@code bytes}: the bit reader loads 64-bit windows @@ -36,12 +51,27 @@ public ReadStream( byte[] buffer, int bytes ) public void reset( byte[] buffer, int bytes ) { reader.reset( buffer, bytes ); + failed = false; } @Override public boolean isWriting() { return false; } @Override public boolean isReading() { return true; } + /** Has a read on this stream failed? Every read after the first refusal refuses. */ + public boolean isFailed() + { + return failed; + } + + // Sets the latch and reports the refusal in one expression, so a refusal + // reads as `return fail();` wherever it occurs. + private boolean fail() + { + failed = true; + return false; + } + // the contracts of the hot operations live in their own methods so the // hot bodies stay small enough for the JIT to inline: an assert's // bytecode is carried even when -ea is absent, and it counts against @@ -64,10 +94,11 @@ private static boolean checkBits64( int bits ) @Override public boolean serializeBits( IntRef value, int bits ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after assert checkBits32( bits ); if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } value.value = reader.readBits( bits ); return true; @@ -76,12 +107,13 @@ public boolean serializeBits( IntRef value, int bits ) @Override public boolean serializeBits64( LongRef value, int bits ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after assert checkBits64( bits ); if ( bits <= 32 ) { if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } value.value = Integer.toUnsignedLong( reader.readBits( bits ) ); } @@ -91,12 +123,12 @@ public boolean serializeBits64( LongRef value, int bits ) // matching the reference macro's composition from two 32-bit operations if ( reader.wouldReadPastEnd( 32 ) ) { - return false; + return fail(); } long lo = Integer.toUnsignedLong( reader.readBits( 32 ) ); if ( reader.wouldReadPastEnd( bits - 32 ) ) { - return false; + return fail(); } long hi = Integer.toUnsignedLong( reader.readBits( bits - 32 ) ); value.value = ( hi << 32 ) | lo; @@ -107,6 +139,7 @@ public boolean serializeBits64( LongRef value, int bits ) @Override public boolean serializeInt( IntRef value, int min, int max ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after assert min <= max; int bits = SerializeUtil.bitsRequired( min, max ); if ( bits == 0 ) @@ -116,12 +149,12 @@ public boolean serializeInt( IntRef value, int min, int max ) } if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } int unsignedValue = reader.readBits( bits ); if ( Integer.compareUnsigned( unsignedValue, max - min ) > 0 ) { - return false; + return fail(); } // add in the unsigned domain: wraps when the range is wider than 2^31 value.value = unsignedValue + min; @@ -131,6 +164,7 @@ public boolean serializeInt( IntRef value, int min, int max ) @Override public boolean serializeInt64( LongRef value, long min, long max ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after assert min <= max; int bits = SerializeUtil.bitsRequired64( min, max ); if ( bits == 0 ) @@ -141,7 +175,7 @@ public boolean serializeInt64( LongRef value, long min, long max ) // one truncation check for the whole value, matching the reference stream method if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } long unsignedValue; if ( bits <= 32 ) @@ -156,7 +190,7 @@ public boolean serializeInt64( LongRef value, long min, long max ) } if ( Long.compareUnsigned( unsignedValue, max - min ) > 0 ) { - return false; + return fail(); } // add in the unsigned domain: wraps when the range is wider than 2^63 value.value = unsignedValue + min; @@ -166,27 +200,35 @@ public boolean serializeInt64( LongRef value, long min, long max ) @Override public boolean serializeInt128( Ref value, Int128Value min, Int128Value max ) { - assert min.compareTo( max ) < 0; + if ( failed ) return false; // the latch: a failed stream refuses everything after + assert min.compareTo( max ) <= 0; UInt128Value unsignedMin = min.toUnsigned(); UInt128Value unsignedMax = max.toUnsigned(); int bits = SerializeUtil.bitsRequired128( unsignedMin, unsignedMax ); + if ( bits == 0 ) + { + value.value = min; // degenerate range: the value IS the range + return true; + } // one truncation check for the whole value, matching the reference stream method if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } UInt128Value offset = readGroups128( bits ); if ( offset.compareUnsigned( unsignedMax.subtract( unsignedMin ) ) > 0 ) { - return false; + return fail(); } // add in the unsigned domain: wraps when the range is wider than 2^127 value.value = Int128Value.fromUnsigned( offset.add( unsignedMin ) ); return true; } - // 32-bit groups, least significant first. The caller has already priced the - // whole value against the stream end. + // 32-bit groups, least significant first, for a width of 1 to 128 bits. + // The caller has already priced the whole value against the stream end and + // has routed the zero-bit degenerate range away: the bit primitive reads + // 1 to 32 bits per group. private UInt128Value readGroups128( int bits ) { long group0 = 0; @@ -245,15 +287,16 @@ public boolean serializeUint64( LongRef value ) @Override public boolean serializeUint128( Ref value ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after // the low 64-bit half first, then the high half — composed from 32-bit // groups with per-group truncation checks, matching the reference macro - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); long a = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); long b = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); long c = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); long d = Integer.toUnsignedLong( reader.readBits( 32 ) ); value.value = new UInt128Value( ( d << 32 ) | c, ( b << 32 ) | a ); return true; @@ -262,9 +305,10 @@ public boolean serializeUint128( Ref value ) @Override public boolean serializeBool( BoolRef value ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after if ( reader.wouldReadPastEnd( 1 ) ) { - return false; + return fail(); } value.value = reader.readBits( 1 ) != 0; return true; @@ -273,9 +317,10 @@ public boolean serializeBool( BoolRef value ) @Override public boolean serializeFloat( FloatRef value ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after if ( reader.wouldReadPastEnd( 32 ) ) { - return false; + return fail(); } // bit transparent: the read returns exactly the bits read — NaN payloads, // signaling NaNs, infinities, negative zero and denormals all pass through @@ -286,16 +331,17 @@ public boolean serializeFloat( FloatRef value ) @Override public boolean serializeDouble( DoubleRef value ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after // two 32-bit groups with per-group truncation checks, matching the // reference macro's composition if ( reader.wouldReadPastEnd( 32 ) ) { - return false; + return fail(); } long lo = Integer.toUnsignedLong( reader.readBits( 32 ) ); if ( reader.wouldReadPastEnd( 32 ) ) { - return false; + return fail(); } long hi = Integer.toUnsignedLong( reader.readBits( 32 ) ); value.value = Double.longBitsToDouble( ( hi << 32 ) | lo ); @@ -305,18 +351,19 @@ public boolean serializeDouble( DoubleRef value ) @Override public boolean serializeCompressedFloat( FloatRef value, float min, float max, float resolution ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after long maxIntegerValue = SerializeUtil.compressedFloatMaxIntegerValue( min, max, resolution ); int bits = SerializeUtil.bitsRequired( 0, (int) maxIntegerValue ); float delta = max - min; if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } int integerValue = reader.readBits( bits ); // reject an integer above maxIntegerValue smuggled into the bit headroom if ( Integer.toUnsignedLong( integerValue ) > maxIntegerValue ) { - return false; + return fail(); } value.value = SerializeUtil.compressedFloatReadValue( integerValue, maxIntegerValue, delta, min ); return true; @@ -325,18 +372,19 @@ public boolean serializeCompressedFloat( FloatRef value, float min, float max, f @Override public boolean serializeBytes( byte[] data, int bytes ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after if ( bytes < 0 ) { - return false; + return fail(); } if ( !serializeAlign() ) { - return false; + return fail(); } // compare in bytes rather than bits, consistent with the reference's bookkeeping if ( bytes > reader.getBitsRemaining() / 8 ) { - return false; + return fail(); } reader.readBytes( data, bytes ); return true; @@ -345,28 +393,34 @@ public boolean serializeBytes( byte[] data, int bytes ) @Override public boolean serializeAlign() { + if ( failed ) return false; // the latch: a failed stream refuses everything after int alignBits = reader.getAlignBits(); if ( reader.wouldReadPastEnd( alignBits ) ) { - return false; + return fail(); } - return reader.readAlign(); + if ( !reader.readAlign() ) + { + return fail(); // the padding was not zero + } + return true; } @Override public boolean serializeString( Ref value, int bufferSize ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after // the length, over [0, bufferSize-1] IntRef length = new IntRef(); if ( !serializeInt( length, 0, bufferSize - 1 ) ) { - return false; + return fail(); } // the bytes, which align byte[] utf8 = new byte[length.value]; if ( !serializeBytes( utf8, length.value ) ) { - return false; + return fail(); } // STANDARD.md, "Readers must refuse malformed string payloads". // Interior NUL first: a zero byte among the transmitted bytes gives the @@ -376,12 +430,12 @@ public boolean serializeString( Ref value, int bufferSize ) { if ( utf8[i] == 0 ) { - return false; + return fail(); } } if ( !SerializeUtil.isValidUtf8( utf8, length.value ) ) { - return false; + return fail(); } value.value = new String( utf8, StandardCharsets.UTF_8 ); return true; @@ -390,10 +444,11 @@ public boolean serializeString( Ref value, int bufferSize ) @Override public boolean serializeWideString( Ref value, int bufferSize ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after IntRef length = new IntRef(); if ( !serializeInt( length, 0, bufferSize - 1 ) ) { - return false; + return fail(); } // each group is one UTF-16 code unit, and malformed payloads are refused // in every build mode: a group above 0xFFFF is not a code unit, an @@ -409,22 +464,22 @@ public boolean serializeWideString( Ref value, int bufferSize ) { if ( reader.wouldReadPastEnd( 32 ) ) { - return false; + return fail(); } int character = reader.readBits( 32 ); if ( Integer.compareUnsigned( character, 0xFFFF ) > 0 ) { - return false; // not a UTF-16 code unit: nothing conforming emits one + return fail(); // not a UTF-16 code unit: nothing conforming emits one } if ( character == 0 ) { - return false; // interior NUL: the two-lengths smuggling primitive + return fail(); // interior NUL: the two-lengths smuggling primitive } if ( havePending ) { if ( character < 0xDC00 || character > 0xDFFF ) { - return false; // high surrogate without its low + return fail(); // high surrogate without its low } output[outputIndex++] = pending; output[outputIndex++] = (char) character; @@ -433,7 +488,7 @@ public boolean serializeWideString( Ref value, int bufferSize ) } if ( character >= 0xDC00 && character <= 0xDFFF ) { - return false; // low surrogate with no high before it + return fail(); // low surrogate with no high before it } if ( character >= 0xD800 && character <= 0xDBFF ) { @@ -445,7 +500,7 @@ public boolean serializeWideString( Ref value, int bufferSize ) } if ( havePending ) { - return false; // the final group is a dangling high surrogate + return fail(); // the final group is a dangling high surrogate } value.value = new String( output, 0, outputIndex ); return true; @@ -454,16 +509,17 @@ public boolean serializeWideString( Ref value, int bufferSize ) @Override public boolean serializeIntRelative( int previous, IntRef current ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after + assert previous >= 0; // the domain: previous is caller state, never off the wire + // the one-bit tier if ( reader.wouldReadPastEnd( 1 ) ) { - return false; + return fail(); } if ( reader.readBits( 1 ) != 0 ) { - // reconstruct in the unsigned domain: wraps near the type maximum - current.value = previous + 1; - return true; + return acceptIntRelative( previous, (long) previous + 1, current ); } // the bounded difference tiers @@ -471,7 +527,7 @@ public boolean serializeIntRelative( int previous, IntRef current ) { if ( reader.wouldReadPastEnd( 1 ) ) { - return false; + return fail(); } if ( reader.readBits( 1 ) != 0 ) { @@ -480,30 +536,44 @@ public boolean serializeIntRelative( int previous, IntRef current ) int bits = SerializeUtil.bitsRequired( tierMin, tierMax ); if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } int unsignedValue = reader.readBits( bits ); if ( Integer.compareUnsigned( unsignedValue, tierMax - tierMin ) > 0 ) { - return false; + return fail(); } - // reconstruct in the unsigned domain: wraps near the type maximum - current.value = previous + ( unsignedValue + tierMin ); - return true; + long difference = Integer.toUnsignedLong( unsignedValue ) + tierMin; + return acceptIntRelative( previous, previous + difference, current ); } } - // the final tier transmits current, not the difference, and the reader - // must check the ordering the absolute form does not carry + // the final tier transmits current, not the difference. Its 32 raw bits + // are unsigned, so a value with the top bit set lies outside the domain + // and the reconstruction check below refuses it. if ( reader.wouldReadPastEnd( 32 ) ) { - return false; + return fail(); + } + long absolute = Integer.toUnsignedLong( reader.readBits( 32 ) ); + return acceptIntRelative( previous, absolute, current ); + } + + // STANDARD.md, "int_relative": every tier reconstructs current in a width + // that cannot wrap — a long here — and the read is refused unless the + // result lies in the domain, 0 to 2^31 - 1 inclusive, and is strictly + // greater than previous. The destination is written only on acceptance. + private boolean acceptIntRelative( int previous, long reconstructed, IntRef current ) + { + if ( reconstructed < 0 || reconstructed > Integer.MAX_VALUE ) + { + return fail(); } - current.value = reader.readBits( 32 ); - if ( current.value <= previous ) + if ( reconstructed <= previous ) { - return false; + return fail(); } + current.value = (int) reconstructed; return true; } @@ -513,6 +583,7 @@ public boolean serializeIntRelative( int previous, IntRef current ) @Override public boolean serializeFixed( LongRef value, int integerBits, int fractionBits, long minUnits, long maxUnits ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after assert integerBits >= 1; assert fractionBits >= 0; int width = integerBits + fractionBits; @@ -537,7 +608,7 @@ public boolean serializeFixed( LongRef value, int integerBits, int fractionBits, { if ( reader.wouldReadPastEnd( bits ) ) { - return false; + return fail(); } offset = Integer.toUnsignedLong( reader.readBits( bits ) ); } @@ -547,12 +618,12 @@ public boolean serializeFixed( LongRef value, int integerBits, int fractionBits, // two stream-level 32-bit operations if ( reader.wouldReadPastEnd( 32 ) ) { - return false; + return fail(); } long lo = Integer.toUnsignedLong( reader.readBits( 32 ) ); if ( reader.wouldReadPastEnd( bits - 32 ) ) { - return false; + return fail(); } long hi = Integer.toUnsignedLong( reader.readBits( bits - 32 ) ); offset = ( hi << 32 ) | lo; @@ -561,7 +632,7 @@ public boolean serializeFixed( LongRef value, int integerBits, int fractionBits, // reject raw values outside [rawMin,rawMax] smuggled into the bit headroom — reject, never clamp if ( Long.compareUnsigned( offset, rawRange ) > 0 ) { - return false; + return fail(); } value.value = rawMin + offset; return true; @@ -570,6 +641,7 @@ public boolean serializeFixed( LongRef value, int integerBits, int fractionBits, @Override public boolean serializeFixed128( Ref value, int integerBits, int fractionBits, long minUnits, long maxUnits ) { + if ( failed ) return false; // the latch: a failed stream refuses everything after assert integerBits >= 1; assert fractionBits >= 0; assert integerBits + fractionBits == 128; @@ -597,34 +669,34 @@ public boolean serializeFixed128( Ref value, int integerBits, int f long group3 = 0; if ( bits <= 32 ) { - if ( reader.wouldReadPastEnd( bits ) ) return false; + if ( reader.wouldReadPastEnd( bits ) ) return fail(); group0 = Integer.toUnsignedLong( reader.readBits( bits ) ); } else if ( bits <= 64 ) { - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); group0 = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( bits - 32 ) ) return false; + if ( reader.wouldReadPastEnd( bits - 32 ) ) return fail(); group1 = Integer.toUnsignedLong( reader.readBits( bits - 32 ) ); } else if ( bits <= 96 ) { - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); group0 = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); group1 = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( bits - 64 ) ) return false; + if ( reader.wouldReadPastEnd( bits - 64 ) ) return fail(); group2 = Integer.toUnsignedLong( reader.readBits( bits - 64 ) ); } else { - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); group0 = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); group1 = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( 32 ) ) return false; + if ( reader.wouldReadPastEnd( 32 ) ) return fail(); group2 = Integer.toUnsignedLong( reader.readBits( 32 ) ); - if ( reader.wouldReadPastEnd( bits - 96 ) ) return false; + if ( reader.wouldReadPastEnd( bits - 96 ) ) return fail(); group3 = Integer.toUnsignedLong( reader.readBits( bits - 96 ) ); } UInt128Value offset = new UInt128Value( ( group3 << 32 ) | group2, ( group1 << 32 ) | group0 ); @@ -632,7 +704,7 @@ else if ( bits <= 96 ) // reject raw values outside [rawMin,rawMax] smuggled into the bit headroom — reject, never clamp if ( offset.compareUnsigned( rawRange ) > 0 ) { - return false; + return fail(); } value.value = Int128Value.fromUnsigned( rawMin.add( offset ) ); return true; diff --git a/src/serialize/SerializeUtil.java b/src/serialize/SerializeUtil.java index 8e15db4..51ebcb7 100644 --- a/src/serialize/SerializeUtil.java +++ b/src/serialize/SerializeUtil.java @@ -10,6 +10,9 @@ public final class SerializeUtil { private SerializeUtil() {} + /** The library version, matching the release tag. */ + public static final String VERSION = "1.1.0"; + /** * The number of bits required to serialize an integer in [min,max]. * The subtraction wraps in the unsigned 32-bit domain, so the full diff --git a/src/serialize/WriteStream.java b/src/serialize/WriteStream.java index 8401de1..b362a07 100644 --- a/src/serialize/WriteStream.java +++ b/src/serialize/WriteStream.java @@ -146,17 +146,23 @@ public boolean serializeInt64( LongRef value, long min, long max ) @Override public boolean serializeInt128( Ref value, Int128Value min, Int128Value max ) { - assert min.compareTo( max ) < 0; + assert min.compareTo( max ) <= 0; assert value.value.compareTo( min ) >= 0; assert value.value.compareTo( max ) <= 0; int bits = SerializeUtil.bitsRequired128( min.toUnsigned(), max.toUnsigned() ); + if ( bits == 0 ) + { + return true; // degenerate range: nothing goes on the wire + } // subtract in the unsigned domain: wraps when the range is wider than 2^127 UInt128Value offset = value.value.toUnsigned().subtract( min.toUnsigned() ); writeGroups128( offset, bits ); return true; } - // 32-bit groups, least significant first: the shared wide-value convention + // 32-bit groups, least significant first, for a width of 1 to 128 bits: the + // shared wide-value convention. The caller routes the zero-bit degenerate + // range away, so every group carries 1 to 32 bits. private void writeGroups128( UInt128Value offset, int bits ) { int group0 = (int) offset.lo; @@ -309,8 +315,8 @@ public boolean serializeWideString( Ref value, int bufferSize ) @Override public boolean serializeIntRelative( int previous, IntRef current ) { + assert previous >= 0; // the domain: 0 to 2^31 - 1, previous and current alike assert previous < current.value; - // subtract in the unsigned domain: wraps when the gap is wider than 2^31 int difference = current.value - previous; boolean oneBit = difference == 1; diff --git a/test/serialize/tests/AllTests.java b/test/serialize/tests/AllTests.java index 550a128..a2d67b1 100644 --- a/test/serialize/tests/AllTests.java +++ b/test/serialize/tests/AllTests.java @@ -1,21 +1,35 @@ package serialize.tests; -/** Runs every suite. Exits nonzero on any failure. Run with -ea: the write-side contracts are asserts. */ +/** + * Runs every suite. Exits nonzero on any failure. + * + * Two shapes, both gates. The default is the checked shape, run with + * {@code -ea}, where the write-side contracts are asserts and the suite must + * exercise them; a forgotten {@code -ea} would pass silently while testing + * nothing on the write side, so it is refused. The release shape is asked for + * by name — {@code java AllTests --release}, with assertions off — and proves + * the read side's refusals bind without them: every obligation STANDARD.md + * places on a reader is a check, never an assert. + */ public final class AllTests { private AllTests() {} public static void main( String[] args ) { + boolean release = args.length > 0 && args[0].equals( "--release" ); boolean assertionsEnabled = false; assert assertionsEnabled = true; - if ( !assertionsEnabled ) + if ( assertionsEnabled == release ) { - System.out.println( "error: run with -ea — the write-side contracts are asserts and the suite must exercise them" ); + System.out.println( release + ? "error: --release is the assertions-off shape — run it without -ea" + : "error: run with -ea — the write-side contracts are asserts and the suite must exercise them" ); System.exit( 2 ); } - System.out.println( "serialize.java test suite" ); + System.out.println( release ? "serialize.java test suite (release shape, assertions off)" + : "serialize.java test suite (checked shape, assertions on)" ); System.out.println(); BitpackerTests.run(); @@ -27,6 +41,7 @@ public static void main( String[] args ) StringTests.run(); MeasureTests.run(); GoldenWireTests.run(); + ConformanceTests.run(); System.exit( Harness.finish() ); } diff --git a/test/serialize/tests/ConformanceTests.java b/test/serialize/tests/ConformanceTests.java new file mode 100644 index 0000000..7410780 --- /dev/null +++ b/test/serialize/tests/ConformanceTests.java @@ -0,0 +1,287 @@ +package serialize.tests; + +import serialize.Int128Value; +import serialize.IntRef; +import serialize.ReadStream; +import serialize.Ref; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.math.BigInteger; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static serialize.tests.Harness.check; +import static serialize.tests.Harness.checkEqual; +import static serialize.tests.Harness.test; + +/** + * The shared conformance corpus, run through this library's reader. + * + * {@code conformance/} is a verbatim vendored copy of the corpus in + * mas-bandwidth/serialize, one file per operation, and CI fails if the two + * diverge. Every vector in it runs here: an accepted vector must decode to + * the stated value and consume the stated number of bits, and a refused + * vector must be refused with the destination left unwritten. Nothing in + * this file regenerates an expectation — the corpus is the authority, and a + * suite that computes its own answers proves only that the port agrees with + * itself. + * + * The corpus directory is read relative to the working directory, which is + * the repository root under {@code make test}. An operation with no runner + * below fails the suite rather than being skipped: a vector that does not + * run is a vector that proves nothing. + */ +final class ConformanceTests +{ + private ConformanceTests() {} + + private static final Path CORPUS = Paths.get( "conformance" ); + + /** A value no vector decodes to, so a written destination is visible after a refusal. */ + private static final int SENTINEL = 0x5E5E5E5E; + + private static final Int128Value SENTINEL_128 = new Int128Value( 0x5E5E5E5E5E5E5E5EL, 0x5E5E5E5E5E5E5E5EL ); + + private static final BigInteger MASK_64 = BigInteger.ONE.shiftLeft( 64 ).subtract( BigInteger.ONE ); + + private static final BigInteger MASK_128 = BigInteger.ONE.shiftLeft( 128 ).subtract( BigInteger.ONE ); + + static void run() + { + List files = corpusFiles(); + test( "conformance: the vendored corpus is present", () -> { + check( !files.isEmpty(), "no vector files under " + CORPUS.toAbsolutePath() ); + } ); + + for ( Path file : files ) + { + List vectors = parse( file ); + String fileName = file.getFileName().toString(); + test( "conformance " + fileName + ": the file holds vectors", () -> { + check( !vectors.isEmpty(), fileName + " parsed to no vectors" ); + } ); + for ( Vector vector : vectors ) + { + test( "conformance " + fileName + ": " + vector.name, () -> vector.run() ); + } + } + } + + private static List corpusFiles() + { + List files = new ArrayList<>(); + if ( !Files.isDirectory( CORPUS ) ) + { + return files; + } + try ( DirectoryStream entries = Files.newDirectoryStream( CORPUS, "*.txt" ) ) + { + for ( Path entry : entries ) + { + files.add( entry ); + } + } + catch ( IOException error ) + { + throw new UncheckedIOException( error ); + } + files.sort( null ); + return files; + } + + // STANDARD.md, "The vector format": text, `#` begins a comment, blank + // lines separate records, each record is `key` and value one per line. + private static List parse( Path file ) + { + List lines; + try + { + lines = Files.readAllLines( file ); + } + catch ( IOException error ) + { + throw new UncheckedIOException( error ); + } + + List vectors = new ArrayList<>(); + Vector current = new Vector(); + for ( String rawLine : lines ) + { + int comment = rawLine.indexOf( '#' ); + String line = ( comment >= 0 ? rawLine.substring( 0, comment ) : rawLine ).trim(); + if ( line.isEmpty() ) + { + if ( current.started() ) + { + vectors.add( current ); + current = new Vector(); + } + continue; + } + int space = line.indexOf( ' ' ); + String key = space < 0 ? line : line.substring( 0, space ); + String value = space < 0 ? "" : line.substring( space + 1 ).trim(); + current.put( key, value ); + } + if ( current.started() ) + { + vectors.add( current ); + } + return vectors; + } + + private static final class Vector + { + String operation; + String name; + final Map params = new LinkedHashMap<>(); + byte[] bytes = new byte[0]; + boolean refused; + String expected; + long consumed; + + boolean started() + { + return operation != null; + } + + void put( String key, String value ) + { + switch ( key ) + { + case "operation": + operation = value; + break; + case "name": + name = value; + break; + case "param": + { + int equals = value.indexOf( '=' ); + check( equals > 0, "malformed param: " + value ); + params.put( value.substring( 0, equals ).trim(), value.substring( equals + 1 ).trim() ); + break; + } + case "bytes": + bytes = hex( value ); + break; + case "expect": + if ( value.equals( "refused" ) ) + { + refused = true; + } + else + { + int equals = value.indexOf( '=' ); + check( equals > 0, "malformed expect: " + value ); + expected = value.substring( equals + 1 ).trim(); + } + break; + case "consumed": + consumed = Long.parseLong( value ); + break; + default: + check( false, "unknown vector key: " + key ); + break; + } + } + + String param( String key ) + { + String value = params.get( key ); + check( value != null, "vector " + name + " has no param " + key ); + return value; + } + + /** The reader's buffer: the vector's bytes plus the 8 bytes of slack the reader loads through. */ + byte[] buffer() + { + byte[] buffer = new byte[bytes.length + 8]; + System.arraycopy( bytes, 0, buffer, 0, bytes.length ); + return buffer; + } + + void run() + { + switch ( operation ) + { + case "int_relative": + runIntRelative(); + break; + case "int128": + runInt128(); + break; + default: + check( false, "the corpus carries operation " + operation + ", which this suite does not run" ); + break; + } + } + + private void runIntRelative() + { + int previous = Integer.parseInt( param( "previous" ) ); + ReadStream reader = new ReadStream( buffer(), bytes.length ); + IntRef current = new IntRef( SENTINEL ); + boolean accepted = reader.serializeIntRelative( previous, current ); + if ( refused ) + { + check( !accepted, "must refuse" ); + checkEqual( current.value, SENTINEL, "destination unwritten after a refusal" ); + return; + } + check( accepted, "must accept" ); + checkEqual( current.value, Long.parseLong( expected ), "decoded value" ); + checkEqual( reader.getBitsProcessed(), consumed, "bits consumed" ); + } + + private void runInt128() + { + Int128Value min = toInt128( new BigInteger( param( "min" ) ) ); + Int128Value max = toInt128( new BigInteger( param( "max" ) ) ); + ReadStream reader = new ReadStream( buffer(), bytes.length ); + Ref value = new Ref<>( SENTINEL_128 ); + boolean accepted = reader.serializeInt128( value, min, max ); + if ( refused ) + { + check( !accepted, "must refuse" ); + check( value.value.equals( SENTINEL_128 ), "destination unwritten after a refusal" ); + return; + } + check( accepted, "must accept" ); + check( fromInt128( value.value ).equals( new BigInteger( expected ) ), + "decoded value: got " + fromInt128( value.value ) + ", expected " + expected ); + checkEqual( reader.getBitsProcessed(), consumed, "bits consumed" ); + } + } + + private static byte[] hex( String text ) + { + String[] pairs = text.isEmpty() ? new String[0] : text.split( "\\s+" ); + byte[] bytes = new byte[pairs.length]; + for ( int i = 0; i < pairs.length; i++ ) + { + bytes[i] = (byte) Integer.parseInt( pairs[i], 16 ); + } + return bytes; + } + + /** The decimal vector value as the library's two's-complement pair. */ + private static Int128Value toInt128( BigInteger value ) + { + BigInteger bits = value.and( MASK_128 ); + return new Int128Value( bits.shiftRight( 64 ).longValue(), bits.and( MASK_64 ).longValue() ); + } + + /** The library's pair back to a signed decimal, for comparison against the vector. */ + private static BigInteger fromInt128( Int128Value value ) + { + return BigInteger.valueOf( value.hi ).shiftLeft( 64 ).add( BigInteger.valueOf( value.lo ).and( MASK_64 ) ); + } +} diff --git a/test/serialize/tests/Int128Tests.java b/test/serialize/tests/Int128Tests.java index a55a82f..084b89a 100644 --- a/test/serialize/tests/Int128Tests.java +++ b/test/serialize/tests/Int128Tests.java @@ -206,6 +206,27 @@ static void run() check( !reader.serializeInt128( readBack, Int128Value.ZERO, Int128Value.fromLong( 200 ) ), "255 in [0,200] refused" ); } ); + test( "int128: a degenerate range is legal and costs zero bits", () -> { + // the corpus vector's bounds: 2^100 + 7, min == max + Int128Value bound = new Int128Value( 0x0000000000000010L, 0x0000000000000007L ); + + byte[] buffer = new byte[16 + 8]; + WriteStream writer = new WriteStream( buffer, 16 ); + check( writer.serializeInt128( new Ref<>( bound ), bound, bound ) ); + writer.flush(); + checkEqual( writer.getBitsProcessed(), 0, "the writer emits nothing" ); + + MeasureStream measure = new MeasureStream(); + check( measure.serializeInt128( new Ref<>( bound ), bound, bound ) ); + checkEqual( measure.getBitsProcessed(), 0, "the measure adds zero" ); + + ReadStream reader = new ReadStream( new byte[8], 0 ); // an empty stream carries it + Ref readBack = new Ref<>( Int128Value.ZERO ); + check( reader.serializeInt128( readBack, bound, bound ) ); + check( readBack.value.equals( bound ), "the value comes from min" ); + checkEqual( reader.getBitsProcessed(), 0, "the reader consumes nothing" ); + } ); + test( "int128: a truncated buffer refuses rather than reading past the end", () -> { ReadStream reader = new ReadStream( new byte[32 + 8], 4 ); // 32 bits available, 128 required Ref readBack = new Ref<>( Int128Value.ZERO ); diff --git a/test/serialize/tests/IntRelativeTests.java b/test/serialize/tests/IntRelativeTests.java index d9b873b..a6a54e4 100644 --- a/test/serialize/tests/IntRelativeTests.java +++ b/test/serialize/tests/IntRelativeTests.java @@ -65,38 +65,70 @@ static void run() check( !reader.serializeIntRelative( 100, current ), "50 after 100 refused" ); } ); - test( "intRelative: gaps wider than 2^31 travel through the unsigned domain", () -> { - byte[] buffer = new byte[8 + 8]; - WriteStream writer = new WriteStream( buffer, 8 ); - int previous = -1000; - int written = Integer.MAX_VALUE; - check( writer.serializeIntRelative( previous, new IntRef( written ) ) ); - writer.flush(); + test( "intRelative: the whole domain travels, from zero to the top", () -> { + // { previous, current } — the widest gap the domain allows, and its top edge + int[][] cases = { + { 0, Integer.MAX_VALUE }, // the absolute tier, end to end + { Integer.MAX_VALUE - 1, Integer.MAX_VALUE }, // the one-bit tier at the top + { Integer.MAX_VALUE - 6, Integer.MAX_VALUE }, // a bounded tier at the top + }; + for ( int[] c : cases ) + { + byte[] buffer = new byte[8 + 8]; + WriteStream writer = new WriteStream( buffer, 8 ); + check( writer.serializeIntRelative( c[0], new IntRef( c[1] ) ) ); + writer.flush(); - ReadStream reader = new ReadStream( buffer, 8 ); - IntRef current = new IntRef(); - check( reader.serializeIntRelative( previous, current ) ); - checkEqual( current.value, written, "round trip across a >2^31 gap" ); + MeasureStream measure = new MeasureStream(); + check( measure.serializeIntRelative( c[0], new IntRef( c[1] ) ) ); + checkEqual( measure.getBitsProcessed(), writer.getBitsProcessed(), "measure agrees" ); + + ReadStream reader = new ReadStream( buffer, (int) writer.getBytesProcessed() ); + IntRef current = new IntRef(); + check( reader.serializeIntRelative( c[0], current ) ); + checkEqual( current.value, c[1], "round trip " + c[0] + " -> " + c[1] ); + } } ); - test( "intRelative: reconstruction near INT32_MAX wraps in the unsigned domain", () -> { - int[] differences = { 1, 5 }; + test( "intRelative: a reconstruction past the domain is refused in every tier", () -> { + // the same bytes decode one step lower: the refusal is on the + // reconstructed value, not on the byte pattern + int[] differences = { 1, 5, 20, 200, 3000, 50000 }; for ( int difference : differences ) { byte[] buffer = new byte[8 + 8]; WriteStream writer = new WriteStream( buffer, 8 ); - int previousWrite = 10; - check( writer.serializeIntRelative( previousWrite, new IntRef( previousWrite + difference ) ) ); + check( writer.serializeIntRelative( 10, new IntRef( 10 + difference ) ) ); writer.flush(); - - ReadStream reader = new ReadStream( buffer, 8 ); - int previous = Integer.MAX_VALUE; // previous + difference exceeds INT32_MAX - IntRef current = new IntRef(); - check( reader.serializeIntRelative( previous, current ) ); - checkEqual( current.value, Integer.MAX_VALUE + difference, "wrapped reconstruction" ); + int bytes = (int) writer.getBytesProcessed(); + + int overflowing = Integer.MAX_VALUE - difference + 1; // reconstructs one past the domain + ReadStream reader = new ReadStream( buffer, bytes ); + IntRef current = new IntRef( 0x5E5E5E5E ); + check( !reader.serializeIntRelative( overflowing, current ), "difference " + difference + " past the domain refused" ); + checkEqual( current.value, 0x5E5E5E5E, "the refusal wrote nothing" ); + + ReadStream inside = new ReadStream( buffer, bytes ); + IntRef accepted = new IntRef(); + check( inside.serializeIntRelative( overflowing - 1, accepted ), "difference " + difference + " one step inside accepted" ); + checkEqual( accepted.value, Integer.MAX_VALUE, "the twin decodes to the domain top" ); } } ); + test( "intRelative: the absolute tier reads its 32 raw bits unsigned", () -> { + // a top-bit-set absolute value is outside the domain, whatever previous is + byte[] buffer = new byte[8 + 8]; + WriteStream writer = new WriteStream( buffer, 8 ); + check( writer.serializeBits( new IntRef( 0 ), 6 ) ); // six false flags + check( writer.serializeBits( new IntRef( 0x80000000 ), 32 ) ); // 2^31, one past the domain + writer.flush(); + + ReadStream reader = new ReadStream( buffer, (int) writer.getBytesProcessed() ); + IntRef current = new IntRef( 0x5E5E5E5E ); + check( !reader.serializeIntRelative( 100, current ), "top bit set refused" ); + checkEqual( current.value, 0x5E5E5E5E, "the refusal wrote nothing" ); + } ); + test( "intRelative: a doctored tier payload out of range refuses", () -> { byte[] buffer = new byte[8 + 8]; WriteStream writer = new WriteStream( buffer, 8 ); diff --git a/test/serialize/tests/StreamTests.java b/test/serialize/tests/StreamTests.java index 3260a8e..f231e38 100644 --- a/test/serialize/tests/StreamTests.java +++ b/test/serialize/tests/StreamTests.java @@ -229,5 +229,105 @@ static void run() check( reader.serializeBits( value, 1 ) ); check( !reader.serializeAlign(), "doctored padding refused" ); } ); + + test( "terminality: a refusal latches, and every later read fails and writes nothing", () -> { + // failure before any consumption + checkTerminal( "past the end at bit zero", failed( reader -> { + check( !reader.serializeBits( new IntRef(), 32 ), "32 bits from an empty stream refused" ); + }, new byte[8], 0 ) ); + + // failure after partial consumption: four bits land, the rest does not + checkTerminal( "past the end mid-stream", failed( reader -> { + check( reader.serializeBits( new IntRef(), 4 ), "the first four bits land" ); + check( !reader.serializeBits( new IntRef(), 32 ), "32 more from one byte refused" ); + }, new byte[9], 1 ) ); + + // failure on range headroom: 255 smuggled into the eight bits of [0,200] + byte[] headroom = new byte[9]; + headroom[0] = (byte) 0xFF; + checkTerminal( "an offset above the range", failed( reader -> { + check( !reader.serializeInt( new IntRef(), 0, 200 ), "255 over [0,200] refused" ); + }, headroom, 1 ) ); + + // failure on alignment: a nonzero padding bit + byte[] alignment = new byte[16]; + WriteStream aligning = new WriteStream( alignment, 8 ); + check( aligning.serializeBits( new IntRef( 1 ), 1 ) ); + check( aligning.serializeAlign() ); + check( aligning.serializeBits( new IntRef( 0x55 ), 8 ) ); + aligning.flush(); + alignment[0] |= (byte) 0x80; + checkTerminal( "nonzero alignment padding", failed( reader -> { + check( reader.serializeBits( new IntRef(), 1 ), "the leading bit lands" ); + check( !reader.serializeAlign(), "doctored padding refused" ); + }, alignment, (int) aligning.getBytesProcessed() ) ); + + // failure on a malformed string: 0xFF appears nowhere in well-formed UTF-8 + byte[] malformed = new byte[72]; + WriteStream writing = new WriteStream( malformed, 64 ); + check( writing.serializeInt( new IntRef( 3 ), 0, 255 ) ); + check( writing.serializeBytes( GoldenWire.bytes( 0xFF, 0xFE, 0xFF ), 3 ) ); + writing.flush(); + checkTerminal( "a malformed string payload", failed( reader -> { + check( !reader.serializeString( new Ref<>( "" ), 256 ), "0xFF payload refused" ); + }, malformed, (int) writing.getBytesProcessed() ) ); + + // failure on int_relative: the one-bit tier reconstructs past the domain + byte[] relative = new byte[9]; + relative[0] = 0x01; + checkTerminal( "an int_relative read past the domain", failed( reader -> { + check( !reader.serializeIntRelative( Integer.MAX_VALUE, new IntRef() ), "past the domain refused" ); + }, relative, 1 ) ); + } ); + + test( "terminality: reset clears the latch", () -> { + byte[] buffer = new byte[9]; + buffer[0] = 0x0F; + ReadStream reader = new ReadStream( buffer, 1 ); + check( !reader.serializeBits( new IntRef(), 32 ), "32 bits from one byte refused" ); + check( reader.isFailed(), "the latch is set" ); + + reader.reset( buffer, 1 ); + check( !reader.isFailed(), "reset cleared the latch" ); + IntRef value = new IntRef(); + check( reader.serializeBits( value, 4 ), "the stream reads again" ); + checkEqual( value.value, 0x0F, "the value after reset" ); + } ); + } + + /** A value no read below decodes to, so a written destination is visible after a refusal. */ + private static final int SENTINEL = 0x5E5E5E5E; + + /** Runs a body that must fail its stream, and hands the failed stream back. */ + private static ReadStream failed( java.util.function.Consumer body, byte[] buffer, int bytes ) + { + ReadStream reader = new ReadStream( buffer, bytes ); + body.accept( reader ); + return reader; + } + + /** + * STANDARD.md, "Failure is terminal": a later read on a failed stream must + * fail, consume no bits and write no destination — including a zero-bit + * read, which consumes nothing and so cannot be caught by the past-end + * check alone. + */ + private static void checkTerminal( String what, ReadStream reader ) + { + check( reader.isFailed(), what + ": the latch is set" ); + long consumed = reader.getBitsProcessed(); + + IntRef destination = new IntRef( SENTINEL ); + check( !reader.serializeBits( destination, 1 ), what + ": a one-bit read fails" ); + checkEqual( destination.value, SENTINEL, what + ": the one-bit read wrote nothing" ); + + check( !reader.serializeInt( destination, 7, 7 ), what + ": a zero-bit read fails" ); + checkEqual( destination.value, SENTINEL, what + ": the zero-bit read wrote nothing" ); + + BoolRef flag = new BoolRef( true ); + check( !reader.serializeBool( flag ), what + ": a bool read fails" ); + check( flag.value, what + ": the bool read wrote nothing" ); + + checkEqual( reader.getBitsProcessed(), consumed, what + ": no bits were consumed after the failure" ); } }