diff --git a/SKILL.md b/SKILL.md index e21ac3a..61a2794 100644 --- a/SKILL.md +++ b/SKILL.md @@ -53,12 +53,18 @@ Every recipe has `schema_version: 1`, `model: {name, kind, revision}` and the rest. **`endpoint`** (any OpenAI-compatible server): `endpoint: {base_url, -api_key_env, dialect: auto|vllm|llamacpp|generic, timeout_s}`, `template` -and `answers`. +api_key_env, dialect: auto|vllm|llamacpp|generic, timeout_s, extra_body}`, +`template` and `answers`. - `template.mode: messages` sends chat messages; put thinking off in `template.kwargs` (`{thinking: false}` for DeepSeek, `{enable_thinking: false}` for Qwen3-class models; names are model-specific, the probe checks that no `` token opens the answer). +- `endpoint.extra_body` is merged into the top level of every request, for + servers that take a switch there rather than as a template argument. The + `generic` dialect sends no template arguments, so on Ollama's `/v1` route + a thinking model needs `extra_body: {reasoning_effort: none}` (`think: + false` is ignored there). A key jevify already sets for the readout + (`max_tokens`, `logprobs`, ...) is refused, not overridden. - `template.mode: raw` sends a completion prompt you write with the model's own control tokens; use it when the chat template would not put the model in its trained regime (the Qwen3-VL reranker recipe is the example). diff --git a/docs/adr/0004-recipes.md b/docs/adr/0004-recipes.md index 8861210..ea0013d 100644 --- a/docs/adr/0004-recipes.md +++ b/docs/adr/0004-recipes.md @@ -32,6 +32,7 @@ hash is the recipe's identity in every answer and every scorecard. | Section | Holds | |---|---| | `model` | Identifier as the backend knows it, revision when known, the adapter kind, and for endpoints the dialect the probe detected. | +| `endpoint` | For endpoint-backed kinds: base URL, the environment variable holding an API key, the dialect, the timeout, and `extra_body`, server-specific fields merged into the top level of every request (never replacing a field the readout sets; omitted from the hash when empty, so older recipes keep theirs). | | `template` | For endpoints: how state, instructions and options are rendered into messages or a raw prompt; where the question goes; the answer-slot suffix; template arguments such as thinking off; whether assistant prefill is used. For encoders: the sequence layout and marker scheme. | | `answers` | The answer tokens for `noul`, the identifier scheme for `choice`, the level tokens for `score`, each with the token ids the probe verified. | | `readout` | The rung of the ladder to use or `auto`; the permutation policy and call count for `choice`. | diff --git a/jevify/adapters/endpoint/client.py b/jevify/adapters/endpoint/client.py index 39927f2..f6a0b3d 100644 --- a/jevify/adapters/endpoint/client.py +++ b/jevify/adapters/endpoint/client.py @@ -5,6 +5,7 @@ import asyncio import random import re +from collections.abc import Mapping from typing import Any import httpx @@ -28,6 +29,7 @@ def __init__( timeout_s: float = 120.0, http: httpx.AsyncClient | None = None, max_retries: int = 3, + extra_body: Mapping[str, Any] | None = None, ) -> None: stripped = base_url.rstrip("/") self.root = stripped[: -len("/v1")] if stripped.endswith("/v1") else stripped @@ -35,6 +37,7 @@ def __init__( self.api_key = api_key self.timeout_s = timeout_s self.max_retries = max_retries + self.extra_body = dict(extra_body or {}) self._http = http or httpx.AsyncClient(timeout=httpx.Timeout(timeout_s)) def _headers(self) -> dict[str, str]: @@ -44,6 +47,8 @@ def _headers(self) -> dict[str, str]: return headers async def _request(self, method: str, url: str, body: dict[str, Any] | None = None) -> Any: + if body is not None and self.extra_body: + body = _merge_extra(body, self.extra_body) attempt = 0 while True: try: @@ -76,6 +81,16 @@ async def aclose(self) -> None: await self._http.aclose() +def _merge_extra(body: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]: + """Add the recipe's extra fields; never let one replace a field the readout set.""" + clash = sorted(set(body) & set(extra)) + if clash: + raise BackendError( + f"endpoint.extra_body may not set {', '.join(clash)}: jevify sets it for the readout" + ) + return {**body, **extra} + + def _retry_delay(response: httpx.Response, attempt: int) -> float: header = response.headers.get("retry-after-ms") or response.headers.get("retry-after") if header: diff --git a/jevify/compose.py b/jevify/compose.py index cdb53dd..bb065f4 100644 --- a/jevify/compose.py +++ b/jevify/compose.py @@ -37,6 +37,7 @@ def build_backend(recipe: Recipe, *, http: Any | None = None) -> Backend: recipe.endpoint.base_url, api_key=api_key, timeout_s=recipe.endpoint.timeout_s, + extra_body=recipe.endpoint.extra_body, http=http, ) return EndpointBackend(client, recipe) @@ -51,6 +52,7 @@ def build_backend(recipe: Recipe, *, http: Any | None = None) -> Backend: recipe.endpoint.base_url, api_key=api_key, timeout_s=recipe.endpoint.timeout_s, + extra_body=recipe.endpoint.extra_body, http=http, ) return RerankBackend(client, recipe) @@ -66,6 +68,7 @@ def build_backend(recipe: Recipe, *, http: Any | None = None) -> Backend: recipe.endpoint.base_url, api_key=api_key, timeout_s=recipe.endpoint.timeout_s, + extra_body=recipe.endpoint.extra_body, http=http, ) from jevify.adapters.embedding.adapter import EndpointEmbedder diff --git a/jevify/recipes/schema.py b/jevify/recipes/schema.py index 1c8a087..0775216 100644 --- a/jevify/recipes/schema.py +++ b/jevify/recipes/schema.py @@ -32,6 +32,10 @@ class EndpointSpec(_Strict): api_key_env: str | None = None dialect: Dialect = "auto" timeout_s: float = Field(default=120.0, gt=0) + # Server-specific request fields sent at the top level of every request, e.g. + # Ollama's `reasoning_effort: "none"`, the only way its /v1 route turns thinking + # off. Left out of the dump when empty so recipes that predate it keep their hash. + extra_body: dict[str, Any] = Field(default_factory=dict, exclude_if=lambda v: not v) class QuestionTemplates(_Strict): diff --git a/tests/fakes/fake_openai_server.py b/tests/fakes/fake_openai_server.py index 1ac31a5..a7e409b 100644 --- a/tests/fakes/fake_openai_server.py +++ b/tests/fakes/fake_openai_server.py @@ -34,6 +34,9 @@ class Behaviour: supports_tokenize: bool = True thinking_kwarg: str | None = "thinking" thinking_default_on: bool = True + # a top-level request field that turns thinking off, as Ollama's /v1 route takes + # `reasoning_effort: "none"` and ignores template arguments + thinking_off_field: tuple[str, Any] | None = None # how the template spells "thinking off": nothing, or a closed empty block (Qwen3.5) thinking_off_style: str = "omit" cache_tail_tokens: int = 0 # trailing prompt tokens never reported cached (llama.cpp) @@ -227,6 +230,9 @@ def _rendered_chat(self, body: dict[str, Any]) -> str: thinking_on = self.behaviour.thinking_default_on if self.behaviour.thinking_kwarg and kwargs.get(self.behaviour.thinking_kwarg) is False: thinking_on = False + off_field = self.behaviour.thinking_off_field + if off_field and body.get(off_field[0]) == off_field[1]: + thinking_on = False last = body["messages"][-1] if last["role"] == "assistant" and body.get("continue_final_message"): if self.behaviour.prefill == "eos_appended": diff --git a/tests/test_endpoint_adapter.py b/tests/test_endpoint_adapter.py index d6649ba..bd943a2 100644 --- a/tests/test_endpoint_adapter.py +++ b/tests/test_endpoint_adapter.py @@ -8,6 +8,7 @@ import httpx import pytest +from jevify.adapters.endpoint.readout import MissingLabelsError from jevify.compose import build_backend from jevify.domain.distribution import Distribution from jevify.domain.questions import ChoiceQuestion, NoulQuestion, Option, State @@ -189,3 +190,53 @@ def test_per_option_choice_composes_yes_no_readouts(fixtures, fake_server_factor assert d.as_mapping()["billing"] == pytest.approx(math.exp(4) / (1 + math.exp(4))) assert answer.off_menu_mass is None assert answer.rung == "top_k" + + +def _with_endpoint(recipe, **update): + return recipe.model_copy(update={"endpoint": recipe.endpoint.model_copy(update=update)}) + + +OLLAMA_LIKE = Behaviour( + scores={" yes": 0.0, " no": -1.0}, + dialect="generic", + thinking_kwarg=None, + thinking_off_field=("reasoning_effort", "none"), +) + + +def test_generic_dialect_cannot_turn_thinking_off_without_extra_body(fixtures, fake_server_factory): + # The failure this field exists for: template arguments never reach a generic + # server, so a thinking model's one-token readout is its first thinking token. + server, http = fake_server_factory(OLLAMA_LIKE) + recipe = load_recipe(fixtures / "recipes" / "fake-vllm.yaml") + backend = build_backend(_with_endpoint(recipe, dialect="generic"), http=http) + + with pytest.raises(MissingLabelsError): + run(backend.evaluate(run(backend.warm(STATE)), [NoulQuestion(id="q", instructions="x")])) + + +def test_extra_body_reaches_every_request_on_the_generic_dialect(fixtures, fake_server_factory): + server, http = fake_server_factory(OLLAMA_LIKE) + recipe = load_recipe(fixtures / "recipes" / "fake-vllm.yaml") + recipe = _with_endpoint(recipe, dialect="generic", extra_body={"reasoning_effort": "none"}) + backend = build_backend(recipe, http=http) + + [answer] = run( + backend.evaluate(run(backend.warm(STATE)), [NoulQuestion(id="q", instructions="x")]) + ) + + assert Distribution.from_logprobs(answer.logprobs).labels == ("false", "true") + assert len(server.requests) == 2 # warm and question + assert all(req["reasoning_effort"] == "none" for req in server.requests) + assert all("chat_template_kwargs" not in req for req in server.requests) + + +def test_extra_body_cannot_override_a_readout_field(fixtures, fake_server_factory): + server, http = fake_server_factory(Behaviour(scores={" yes": 0.0, " no": -1.0})) + recipe = load_recipe(fixtures / "recipes" / "fake-vllm.yaml") + recipe = _with_endpoint(recipe, extra_body={"max_tokens": 64}) + backend = build_backend(recipe, http=http) + + with pytest.raises(BackendError, match="max_tokens"): + run(backend.warm(STATE)) + assert server.requests == [] # refused before anything was sent diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 79a20be..3aa13b0 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -181,3 +181,28 @@ def test_render_identifier_choice_is_a_single_part() -> None: assert rendered.composition == "single" assert len(rendered.parts) == 1 assert rendered.parts[0].labels == ("x", "y") + + +def test_extra_body_is_part_of_the_recipe_and_its_hash(tmp_path: Path) -> None: + doc = yaml.safe_load((FIXTURES / "fake-vllm.yaml").read_text()) + plain = tmp_path / "plain.yaml" + plain.write_text(yaml.safe_dump(doc)) + doc["endpoint"]["extra_body"] = {"reasoning_effort": "none"} + extra = tmp_path / "extra.yaml" + extra.write_text(yaml.safe_dump(doc)) + + assert load_recipe(plain).endpoint.extra_body == {} + assert load_recipe(extra).endpoint.extra_body == {"reasoning_effort": "none"} + assert recipe_hash(load_recipe(plain)) != recipe_hash(load_recipe(extra)) + + +def test_a_recipe_without_extra_body_keeps_the_hash_it_had_before_the_field_existed() -> None: + # Scorecards cite recipe hashes; a new optional field must not re-hash old recipes. + # Literals recorded on the parent commit, before `endpoint.extra_body` was added. + shipped = Path(__file__).parent.parent / "recipes" / "gemma-4-e4b-it.llamacpp.yaml" + assert recipe_hash(load_recipe(FIXTURES / "fake-vllm.yaml")) == ( + "134663112a13f27e6869011075c4eb14dabd9bb8fa8321a883731f42db0bde0e" + ) + assert recipe_hash(load_recipe(shipped)) == ( + "22181ad02a27f7c7958d40132340cef25e7e9b0c1d32a068b330745a16054bad" + )