From a20cfbb710e2c1264df6273171e9c7989a42eea9 Mon Sep 17 00:00:00 2001 From: Felipe Infante de Castro <18445184+fidecastro@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:37:53 -0400 Subject: [PATCH] =?UTF-8?q?Confidence:=20Jev's=20peak=20statistic,=20(n?= =?UTF-8?q?=C2=B7peak=20=E2=88=92=201)/(n=20=E2=88=92=201),=20not=20normal?= =?UTF-8?q?ized=20entropy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TypeSafe defines confidence on choice and score answers as (n × peak − 1) / (n − 1): 0 when uniform, 1 when certain, with the worked example [0.90, 0.06, 0.04] → 0.85. jevify served one minus the normalized entropy under the same field name, which is systematically lower (0.64 on that example) and would have moved a client's threshold decisions. A reader of the NVIDIA forum thread caught it. The formula changes in the one statistics chokepoint; the tests assert TypeSafe's literals. ADR-0003 gains D8, invariants §5 is amended and the change is recorded in §7 with the previous wording. Committed scorecards that predate this keep their recorded numbers, as the ADR says. Co-Authored-By: Claude Fable 5.1 --- SKILL.md | 5 ++++- docs/00-invariants.md | 3 ++- docs/adr/0003-jev-compatible-api.md | 33 ++++++++++++++++++++++++++--- docs/adr/README.md | 2 +- jevify/domain/distribution.py | 11 +++++----- tests/test_distribution.py | 11 +++++++--- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/SKILL.md b/SKILL.md index bfe4465..e21ac3a 100644 --- a/SKILL.md +++ b/SKILL.md @@ -90,7 +90,10 @@ sequence-classification head; all three question types; semantics `readout`. ## Reading an answer Jev's fields come first (`choice`, `confidence`, `probabilities`; `score`, -`legend`; `noul`). Everything jevify adds is under `x_jevify`: +`legend`; `noul`). `confidence` is Jev's statistic, `(n × peak − 1) / (n − 1)` +over the options or levels: 0 when uniform, 1 when certain, and the number +TypeSafe's published thresholds refer to. Everything jevify adds is under +`x_jevify`: | Field | Meaning | |---|---| diff --git a/docs/00-invariants.md b/docs/00-invariants.md index 1ae774d..40fb54f 100644 --- a/docs/00-invariants.md +++ b/docs/00-invariants.md @@ -126,7 +126,7 @@ API response may claim more than it permits. Rules that follow: -- **Confidence** is a statistic of the returned distribution: one minus its normalized entropy for choice and score, distance from one half for noul. It is not an independent estimate that the decision is right, and copy may not describe it as one. +- **Confidence** is Jev's statistic of the returned distribution: `(n × peak − 1) / (n − 1)` over the n options or levels for choice and score, so a uniform distribution scores 0 and a certain one scores 1 (TypeSafe's published definition). Jev's noul answers carry no confidence; jevify reports the distance of P(true) from one half, scaled to [0, 1], under `x_jevify` only. It is not an independent estimate that the decision is right, and copy may not describe it as one. - **"Faster"** may be claimed only against a measured direct call on the same backend, same model, same state, with the measurement in a scorecard. - **"Accuracy"** names its suite and the suite's hash. A suite whose expected answers came from a teacher model measures agreement with that teacher, and copy says so. - **A fine-tuned result is never compared with a zero-shot result in one table** without the words fine-tuned and zero-shot in the row. @@ -150,6 +150,7 @@ number and the previous wording. The edit and the ADR land in the same commit. | Date | Invariant | ADR | Change | |---|---|---|---| | 2026-09-19 | all | ADR-0001 | Adopted. | +| 2026-09-23 | §5, confidence rule | ADR-0003 D8 | Confidence is Jev's peak statistic, `(n × peak − 1) / (n − 1)`. Previous wording: "a statistic of the returned distribution: one minus its normalized entropy for choice and score, distance from one half for noul". The entropy statistic disagreed with Jev's published definition, so the SDK-facing `confidence` field did not carry Jev's meaning. | ## 8. Document map diff --git a/docs/adr/0003-jev-compatible-api.md b/docs/adr/0003-jev-compatible-api.md index 05fdff1..70b31bb 100644 --- a/docs/adr/0003-jev-compatible-api.md +++ b/docs/adr/0003-jev-compatible-api.md @@ -1,6 +1,7 @@ # ADR-0003 — Jev's evaluate call is the API; extensions are namespaced; the SDK is the acceptance test -- **Status:** accepted (2026-09-19). +- **Status:** accepted (2026-09-19); D8 added 2026-09-23: `confidence` is + Jev's peak statistic, amending `docs/00-invariants.md` §5. - **Extends:** ADR-0001 D2. Implements INV-8; constrained by INV-4, INV-9. ## Context @@ -77,6 +78,31 @@ implementation, and a structure guard forbids it from touching the port. The library is the product; the server is one caller of it, and a program that imports jevify gets identical behaviour without HTTP. +### D8 — `confidence` is Jev's statistic, computed Jev's way + +Jev's `confidence` on a `choice` or `score` answer is +`(n × peak − 1) / (n − 1)`, where `n` is the number of options or levels and +`peak` is the largest probability: 0 for a uniform distribution, 1 for a +certain one. This is TypeSafe's published definition +(, worked example +`[0.90, 0.06, 0.04] → 0.85`), and it is what the thresholds TypeSafe +recommends to clients (act above 0.9, review between 0.5 and 0.9, route to +a human below 0.5) are calibrated against. Jev's `noul` answers carry no +confidence. + +jevify computes exactly this in the one statistics chokepoint +(`jevify/domain/distribution.py`) and nowhere else. Until 2026-09-23 it +served one minus the normalized entropy under Jev's field name, which is +systematically lower (`[0.90, 0.06, 0.04]` gave 0.64) and would have moved +a client's threshold decisions; a reader of the NVIDIA forum thread caught +it. For `noul`, jevify keeps its own statistic (distance of P(true) from one +half, scaled to [0, 1]) under `x_jevify.confidence`, never on Jev's field. + +Scorecards under `docs/evidence/` whose `run_at` precedes 2026-09-23 record +the entropy statistic in their wrong-case column; the reliability table bins +on the top probability and is unaffected. They are not rewritten, because a +number in a document comes from a run. + ## Consequences - Compatibility is tested, not asserted. A client written for Jev runs @@ -84,8 +110,9 @@ that imports jevify gets identical behaviour without HTTP. - The `x_jevify` namespace grows without ever touching Jev's fields. - The explicit ingest introduces server-side state and its expiry; the stateless route remains the default and the ingest is opt-in. -- Jev's `confidence` is served with Jev's meaning, a statistic of the - distribution, and the claims contract forbids describing it otherwise. +- Jev's `confidence` is served with Jev's meaning and Jev's formula (D8), a + statistic of the distribution, and the claims contract forbids describing + it otherwise. ## Rejected alternatives diff --git a/docs/adr/README.md b/docs/adr/README.md index fd7d75a..a3e1516 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -14,7 +14,7 @@ later documents can cite "ADR-0002 D3". |---|---|---| | [0001](0001-founding-choices.md) | Founding choices: name, scope, licence, toolchain, conventions | Accepted (2026-09-19); D3 amended 2026-09-20: GPL-3.0 to MIT | | [0002](0002-backend-port-and-adapter-kinds.md) | One backend port, four adapter kinds, a capability probe, a readout ladder | Accepted (2026-09-19) | -| [0003](0003-jev-compatible-api.md) | Jev's evaluate call is the API; extensions are namespaced; the SDK is the acceptance test | Accepted (2026-09-19) | +| [0003](0003-jev-compatible-api.md) | Jev's evaluate call is the API; extensions are namespaced; the SDK is the acceptance test | Accepted (2026-09-19); D8 added 2026-09-23: `confidence` is Jev's peak statistic, amends invariants §5 | | [0004](0004-recipes.md) | Recipes: the hashed file that is the only way a model is asked | Accepted (2026-09-19) | | [0005](0005-evaluation-evidence-and-calibration.md) | Suites, scorecards, committed evidence, post-hoc calibration | Accepted (2026-09-19) | | [0006](0006-toolchain-dependency-budget-and-portability.md) | Toolchain, dependency budget and portability: five light core packages, extras, argparse, uv/pipx, Python 3.12–3.14, no lock-in | Accepted (2026-09-19); makes ADR-0001 D4 precise | diff --git a/jevify/domain/distribution.py b/jevify/domain/distribution.py index 2529aea..519565b 100644 --- a/jevify/domain/distribution.py +++ b/jevify/domain/distribution.py @@ -6,7 +6,8 @@ score-semantics contract in docs/00-invariants.md §5: - a distribution is a softmax over the answer set, so it sums to one; -- confidence for choice and score is one minus the normalized Shannon entropy; +- confidence for choice and score is Jev's peak statistic, (n * peak - 1) / (n - 1): + 0 for a uniform distribution, 1 for a certain one (TypeSafe's published definition); - confidence for noul is the distance of P(true) from one half, scaled to [0, 1]; - the score expectation is the probability-weighted level index. @@ -81,11 +82,11 @@ def confidence(self, kind: QuestionKind) -> float: """A statistic of this distribution, never an estimate that the decision is right.""" if kind == "noul": return min(1.0, max(0.0, abs(self.probabilities[-1] - 0.5) * 2.0)) - k = len(self.probabilities) - if k < 2: + n = len(self.probabilities) + if n < 2: return 1.0 - entropy = -sum(p * math.log(p) for p in self.probabilities if p > 0.0) - return min(1.0, max(0.0, 1.0 - entropy / math.log(k))) + peak = max(self.probabilities) + return min(1.0, max(0.0, (n * peak - 1.0) / (n - 1.0))) def expectation(self) -> float: """Probability-weighted level index: the Jev `score` value.""" diff --git a/tests/test_distribution.py b/tests/test_distribution.py index d3fba36..f5f4df7 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -36,19 +36,24 @@ def test_label_mass_sums_over_token_set() -> None: assert d.probabilities[1] == pytest.approx(1 / 9) -def test_choice_confidence_is_one_minus_normalized_entropy() -> None: +def test_choice_confidence_is_jevs_peak_statistic() -> None: + """Jev: confidence = (n * peak - 1) / (n - 1); 0 when uniform, 1 when certain.""" assert Distribution(("a", "b"), (0.5, 0.5)).confidence("choice") == pytest.approx(0.0) assert Distribution(("a", "b"), (1.0, 0.0)).confidence("choice") == pytest.approx(1.0) uniform4 = Distribution(("a", "b", "c", "d"), (0.25,) * 4) assert uniform4.confidence("choice") == pytest.approx(0.0) - # H([0.9, 0.1]) = 0.32508 nats; / ln 2 = 0.46899; 1 - that = 0.53101 + # TypeSafe's worked example: [0.90, 0.06, 0.04] -> (3 * 0.90 - 1) / 2 = 0.85 + documented = Distribution(("a", "b", "c"), (0.90, 0.06, 0.04)) + assert documented.confidence("choice") == pytest.approx(0.85) + # Two options: (2 * 0.9 - 1) / 1 = 0.8 skewed = Distribution(("a", "b"), (0.9, 0.1)) - assert skewed.confidence("choice") == pytest.approx(0.5310, abs=1e-3) + assert skewed.confidence("choice") == pytest.approx(0.8) def test_score_confidence_uses_the_same_definition_as_choice() -> None: d = Distribution(("0", "1", "2"), (0.05, 0.86, 0.09)) assert d.confidence("score") == pytest.approx(d.confidence("choice")) + assert d.confidence("score") == pytest.approx((3 * 0.86 - 1) / 2) def test_noul_confidence_is_distance_from_half() -> None: