diff --git a/docs/cookbook/02-models.md b/docs/cookbook/02-models.md index 573f64f..801b577 100644 --- a/docs/cookbook/02-models.md +++ b/docs/cookbook/02-models.md @@ -25,6 +25,7 @@ for spec in ( "openrouter/anthropic/claude-haiku-4.5", "openai/gpt-4o-mini", "novita/moonshotai/kimi-k3", + "cheaperinference/gpt-5.4-mini", "ollama/llama3.1", "mock/anything", ): @@ -40,12 +41,13 @@ print(model.invoke("say hi").content) {'spec': 'openrouter/anthropic/claude-haiku-4.5', 'backend': 'openrouter', 'model': 'anthropic/claude-haiku-4.5'} {'spec': 'openai/gpt-4o-mini', 'backend': 'openai', 'model': 'gpt-4o-mini'} {'spec': 'novita/moonshotai/kimi-k3', 'backend': 'novita', 'model': 'moonshotai/kimi-k3'} +{'spec': 'cheaperinference/gpt-5.4-mini', 'backend': 'cheaperinference', 'model': 'gpt-5.4-mini'} {'spec': 'ollama/llama3.1', 'backend': 'ollama', 'model': 'llama3.1'} {'spec': 'mock/anything', 'backend': 'mock', 'model': 'anything'} hello from a scripted model ``` -There are six backends: +There are seven backends: | backend | credential | what it is for | | --- | --- | --- | @@ -53,6 +55,7 @@ There are six backends: | `openrouter` | `OPENROUTER_API_KEY` | one key, most vendors, per-call cost in the response | | `openai` | `OPENAI_API_KEY` | the OpenAI API directly, or any endpoint via `OPENAI_BASE_URL` | | `novita` | `NOVITA_API_KEY` | Novita's own endpoint, token counts but no per-call cost | +| `cheaperinference` | `CHEAPER_INFERENCE_API_KEY` | Cheaper Inference's own endpoint, bare model ids, no per-call cost read | | `ollama` | none — a local server | models on your own machine, free and offline | | `mock` | none | a scripted test double | @@ -66,8 +69,8 @@ needs it. Asking for `claude-cli` therefore does not require `langchain-openai`, for `openrouter` does not require the Claude CLI to be installed. A missing optional dependency fails for the backend that wanted it and nothing else. -`openrouter`, `openai`, `novita` and `ollama` all speak the OpenAI wire format and share one -base class, so they behave identically on everything except money and routing: same +`openrouter`, `openai`, `novita`, `cheaperinference` and `ollama` all speak the OpenAI wire +format and share one base class, so they behave identically on everything except money and routing: same `bind_tools`, same `with_structured_output`, same streaming and async, same retry policy, same usage envelope. @@ -96,7 +99,7 @@ except UnknownBackendError as exc: ('openrouter', 'openai/gpt-4o-mini:floor') ('claude-cli', 'anthropic/claude-haiku-4.5') ('openai', 'gpt-4o-mini') -UnknownBackendError: unknown backend 'opnerouter' in spec 'opnerouter/openai/gpt-4o-mini'; expected one of: claude-cli, openrouter, openai, novita, ollama, mock — or a bare model name for the claude-cli default +UnknownBackendError: unknown backend 'opnerouter' in spec 'opnerouter/openai/gpt-4o-mini'; expected one of: claude-cli, openrouter, openai, novita, cheaperinference, ollama, mock — or a bare model name for the claude-cli default ``` Only the first segment is a backend, because OpenRouter model ids are themselves @@ -125,10 +128,11 @@ model through the broker is still `openrouter/openai/gpt-4o-mini`. ```console $ grapharc models -backends: claude-cli, openrouter, openai, novita, ollama, mock +backends: claude-cli, openrouter, openai, novita, cheaperinference, ollama, mock openrouter key: openai key: novita key: +cheaper inference key: ollama url: http://localhost:11434/v1 examples: @@ -137,6 +141,7 @@ examples: openrouter/openai/gpt-4o-mini:floor cheapest provider for that model openai/gpt-4o-mini the OpenAI API directly, your key novita/moonshotai/kimi-k3 Novita's own endpoint, your key + cheaperinference/gpt-5.4-mini Cheaper Inference's own endpoint, your key ollama/llama3.1 a local server, no key and no bill grapharc models --check probes which of these this machine can use @@ -164,17 +169,19 @@ keys at all: ```console $ grapharc models --check -claude-cli usable 'claude' on PATH at ~/.local/bin/claude - credential: claude subscription login (no API key) -openrouter unusable no API key (set OPENROUTER_API_KEY, or add one to .env) - credential: -openai unusable no API key (set OPENAI_API_KEY, or add one to .env) - credential: -novita unusable no API key (set NOVITA_API_KEY, or add one to .env) - credential: -ollama usable local server at http://localhost:11434/v1 - credential: none needed (local server) -mock usable scripted test double; never reaches a provider +claude-cli usable 'claude' on PATH at ~/.local/bin/claude + credential: claude subscription login (no API key) +openrouter unusable no API key (set OPENROUTER_API_KEY, or add one to .env) + credential: +openai unusable no API key (set OPENAI_API_KEY, or add one to .env) + credential: +novita unusable no API key (set NOVITA_API_KEY, or add one to .env) + credential: +cheaperinference unusable no API key (set CHEAPER_INFERENCE_API_KEY, or add one to .env) + credential: +ollama usable local server at http://localhost:11434/v1 + credential: none needed (local server) +mock usable scripted test double; never reaches a provider local probe only — no provider was contacted, so a configured key is not a validated one. diff --git a/docs/cookbook/06-serving-and-ops.md b/docs/cookbook/06-serving-and-ops.md index 332d4df..aa578e2 100644 --- a/docs/cookbook/06-serving-and-ops.md +++ b/docs/cookbook/06-serving-and-ops.md @@ -1903,15 +1903,19 @@ model: anthropic/claude-haiku-4.5 ```console $ grapharc models --check -claude-cli usable 'claude' on PATH at /home/you/.local/bin/claude - credential: claude subscription login (no API key) -openrouter unusable no API key (set OPENROUTER_API_KEY, or add one to .env) - credential: -openai unusable no API key (set OPENAI_API_KEY, or add one to .env) - credential: -ollama usable local server at http://localhost:11434/v1 - credential: none needed (local server) -mock usable scripted test double; never reaches a provider +claude-cli usable 'claude' on PATH at /home/you/.local/bin/claude + credential: claude subscription login (no API key) +openrouter unusable no API key (set OPENROUTER_API_KEY, or add one to .env) + credential: +openai unusable no API key (set OPENAI_API_KEY, or add one to .env) + credential: +novita unusable no API key (set NOVITA_API_KEY, or add one to .env) + credential: +cheaperinference unusable no API key (set CHEAPER_INFERENCE_API_KEY, or add one to .env) + credential: +ollama usable local server at http://localhost:11434/v1 + credential: none needed (local server) +mock usable scripted test double; never reaches a provider local probe only — no provider was contacted, so a configured key is not a validated one. diff --git a/docs/deep-dive.md b/docs/deep-dive.md index 06dff29..4615849 100644 --- a/docs/deep-dive.md +++ b/docs/deep-dive.md @@ -256,7 +256,7 @@ A stable system is not one that claims to have no edges — it is one whose edge - **`.env` and `grapharc.toml` follow the same discovery rule: the working directory, and nowhere else.** Neither searches parent directories — a run must not be governed by a file you did not know about, and must not be *billed* to one either. **This is a behaviour change:** the credential loader used to walk up to `/`, so a `.env` in an ancestor directory (a `$HOME` one on a shared box, a client project one above a demo checkout) was picked up silently. If you relied on that, move the file into the directory you run from, `export` the variable, or pass `env_file=` to name it explicitly. A real environment variable still beats any file. - **`grapharc run` has no budget unless you give it one.** Set any of `--max-tokens`, `--max-iterations`, `--max-seconds`, or `--max-concurrency`; without them each dimension is unlimited and the gate admits a topology of any worst-case cost. -**Verified this pass:** `pytest` → green, 2,408 selected and 13 deselected (the live ones); `ruff check .` clean; all eight `grapharc demo` stages green, plus the `trace` / `metrics` / `viz` / `replay` tour against a freshly recorded demo trace; the wheel builds and imports all submodules in a clean virtualenv with `[all]`, and `0.1.8` on PyPI is that wheel. The counts are a snapshot, not a property of the project — `pytest` re-derives them in one command, which is the only reason they are quoted, and `tests/test_deep_dive.py` fails this line rather than letting it drift. +**Verified this pass:** `pytest` → green, 2,424 selected and 14 deselected (the live ones); `ruff check .` clean; all eight `grapharc demo` stages green, plus the `trace` / `metrics` / `viz` / `replay` tour against a freshly recorded demo trace; the wheel builds and imports all submodules in a clean virtualenv with `[all]`, and `0.1.8` on PyPI is that wheel. The counts are a snapshot, not a property of the project — `pytest` re-derives them in one command, which is the only reason they are quoted, and `tests/test_deep_dive.py` fails this line rather than letting it drift. [ROADMAP.md](../ROADMAP.md) tracks what is built and what is not, item by item. diff --git a/grapharc/cli/main.py b/grapharc/cli/main.py index c815f92..0bb6801 100644 --- a/grapharc/cli/main.py +++ b/grapharc/cli/main.py @@ -464,6 +464,7 @@ def _cmd_approve(args: argparse.Namespace) -> int: def _cmd_models(args: argparse.Namespace) -> int: from grapharc.gateway import ( + cheaper_inference_api_key, describe, novita_api_key, ollama_base_url, @@ -515,6 +516,7 @@ def _cmd_models(args: argparse.Namespace) -> int: "openrouter/openai/gpt-4o-mini:floor": "cheapest provider for that model", "openai/gpt-4o-mini": "the OpenAI API directly, your key", "novita/moonshotai/kimi-k3": "Novita's own endpoint, your key", + "cheaperinference/gpt-5.4-mini": "Cheaper Inference's own endpoint, your key", "ollama/llama3.1": "a local server, no key and no bill", } payload = { @@ -524,6 +526,7 @@ def _cmd_models(args: argparse.Namespace) -> int: "openrouter_key": redact(openrouter_api_key()), "openai_key": redact(openai_api_key()), "novita_key": redact(novita_api_key()), + "cheaper_inference_key": redact(cheaper_inference_api_key()), # An address, not a secret: it is printed whole, and it is where a # request would go rather than proof that anything is listening. "ollama_base_url": ollama_base_url(), @@ -536,6 +539,7 @@ def _cmd_models(args: argparse.Namespace) -> int: style.kv("openrouter key", redact(openrouter_api_key())), style.kv("openai key", redact(openai_api_key())), style.kv("novita key", redact(novita_api_key())), + style.kv("cheaper inference key", redact(cheaper_inference_api_key())), style.kv("ollama url", ollama_base_url(), tint=style.accent), "", style.heading("examples:"), diff --git a/grapharc/cli/probe.py b/grapharc/cli/probe.py index 703c3b7..3e6b415 100644 --- a/grapharc/cli/probe.py +++ b/grapharc/cli/probe.py @@ -21,9 +21,10 @@ from grapharc.cli import style #: The three columns every row prints, in characters. Constants, not measurements -#: of the terminal: `tests/test_cli.py` asserts on `"bedrock unprobed"`, and -#: the page in `docs/cookbook/02-models.md` shows this table verbatim. -BACKEND_COLUMN = 12 +#: of the terminal: `tests/test_cli.py` asserts on `"bedrock unprobed"`, +#: and the page in `docs/cookbook/02-models.md` shows this table verbatim. The +#: backend column fits the longest name, `cheaperinference`, plus one space. +BACKEND_COLUMN = 17 MARK_COLUMN = 9 # A backend that is a test double rather than a provider. `--check` exits @@ -115,6 +116,26 @@ def _probe_novita() -> dict[str, Any]: } +def _probe_cheaperinference() -> dict[str, Any]: + from grapharc.gateway import cheaper_inference_api_key, redact + + key = cheaper_inference_api_key() + has_dependency = importlib.util.find_spec("langchain_openai") is not None + missing = [] + if not key: + missing.append("no API key (set CHEAPER_INFERENCE_API_KEY, or add one to .env)") + if not has_dependency: + missing.append("langchain-openai not installed (uv sync --extra cheaperinference)") + return { + "backend": "cheaperinference", + "kind": KIND_PROVIDER, + "usable": bool(key) and has_dependency, + "credential": redact(key), + "detail": "; ".join(missing) or "api key configured and langchain-openai installed", + "checked": "credential presence only; no request was sent to api.cheaperinference.com", + } + + def _probe_ollama() -> dict[str, Any]: from grapharc.gateway import ollama_base_url @@ -173,6 +194,7 @@ def probe_backends(*, claude_path: str = "claude") -> list[dict[str, Any]]: "openrouter": _probe_openrouter, "openai": _probe_openai, "novita": _probe_novita, + "cheaperinference": _probe_cheaperinference, "ollama": _probe_ollama, "mock": _probe_mock, } diff --git a/grapharc/gateway/__init__.py b/grapharc/gateway/__init__.py index 47a9c14..541efeb 100644 --- a/grapharc/gateway/__init__.py +++ b/grapharc/gateway/__init__.py @@ -7,6 +7,7 @@ get_model("openrouter/anthropic/claude-sonnet-4.5") # many providers, one key get_model("openai/gpt-4o-mini") # OPENAI_API_KEY get_model("novita/moonshotai/kimi-k3") # NOVITA_API_KEY + get_model("cheaperinference/gpt-5.4-mini") # CHEAPER_INFERENCE_API_KEY get_model("ollama/llama3.1") # local server, no key get_model("mock/x", responses=[...]) # deterministic tests @@ -17,12 +18,14 @@ get_model(spec, cost_ceiling_usd=0.25) # raises when passed get_model(spec, spend=shared_meter) # one ceiling, many models -The four OpenAI-wire backends (`openrouter`, `openai`, `novita`, `ollama`) are -imported lazily — they need `langchain-openai`, which is an optional extra. +The five OpenAI-wire backends (`openrouter`, `openai`, `novita`, +`cheaperinference`, `ollama`) are imported lazily — they need +`langchain-openai`, which is an optional extra. """ from grapharc.gateway.claude_cli import ClaudeCodeCLIChatModel from grapharc.gateway.config import ( + cheaper_inference_api_key, novita_api_key, ollama_api_key, ollama_base_url, @@ -64,6 +67,7 @@ "TransientGatewayError", "UnknownBackendError", "call_with_retry", + "cheaper_inference_api_key", "describe", "different_providers", "get_model", @@ -90,6 +94,8 @@ "OpenAIError": "openai", "NovitaChatModel": "novita", "NovitaError": "novita", + "CheaperInferenceChatModel": "cheaperinference", + "CheaperInferenceError": "cheaperinference", "OllamaChatModel": "ollama", "OllamaError": "ollama", } diff --git a/grapharc/gateway/cheaperinference.py b/grapharc/gateway/cheaperinference.py new file mode 100644 index 0000000..5c8d71f --- /dev/null +++ b/grapharc/gateway/cheaperinference.py @@ -0,0 +1,68 @@ +"""Cheaper Inference backend — an OpenAI-compatible gateway, one key. + +`cheaperinference/gpt-5.4-mini` reaches Cheaper Inference's own +OpenAI-compatible endpoint (`https://api.cheaperinference.com/v1`), not +api.openai.com, so this builds on `OpenAICompatChatModel` the same way +`novita.py` and `openrouter.py` do rather than on `openai.py`: the endpoint is +fixed, not an override of OpenAI's own. + +**Model ids here are bare** — `gpt-5.4-mini`, `claude-sonnet-5`, +`deepseek-v4-flash` — with no `author/` prefix, unlike OpenRouter and Novita. +So `vendor()` in `registry.py` has no author to read, and every spec on this +backend reads as the vendor `cheaperinference`. `BACKEND_VENDOR` deliberately +does not map it to one lab, because the gateway serves several. The +consequence for `different_providers` is the re-seller limit its docstring +already names: two `cheaperinference/…` specs always read as correlated, even +across labs, and `cheaperinference/gpt-5.4-mini` against `openai/gpt-5.4-mini` +reads as independent although both name the same OpenAI model. + +**This backend reads no per-call cost.** `_provider_cost` is the base class's +`None`, so it is the OpenAI one in that respect: `cost_ceiling_usd` counts +calls in `SpendMeter.unpriced_calls` unless a caller supplies +`price_per_million=`. +""" + +from __future__ import annotations + +from typing import Any + +from grapharc.gateway.config import cheaper_inference_api_key +from grapharc.gateway.openai_compat import OpenAICompatChatModel + +CHEAPER_INFERENCE_BASE_URL = "https://api.cheaperinference.com/v1" + + +class CheaperInferenceError(Exception): + """The Cheaper Inference backend could not be constructed or used.""" + + +class CheaperInferenceChatModel(OpenAICompatChatModel): + """A LangChain chat model over Cheaper Inference's OpenAI-compatible endpoint.""" + + def __init__(self, model: str, /, **kwargs: Any) -> None: + api_key = kwargs.pop("api_key", None) or cheaper_inference_api_key() + if not api_key: + raise CheaperInferenceError( + "No Cheaper Inference API key found. Set CHEAPER_INFERENCE_API_KEY " + "in the environment, or add one of CHEAPER_INFERENCE_API_KEY / " + "cheaper-inference-api-key to a .env file." + ) + # One retry layer, not two — same reasoning as the OpenRouter backend. + kwargs.setdefault("max_retries", 0) + super().__init__( + model=model, + api_key=api_key, + base_url=kwargs.pop("base_url", None) or CHEAPER_INFERENCE_BASE_URL, + **kwargs, + ) + + @property + def _llm_type(self) -> str: + return "grapharc-cheaperinference" + + +__all__ = [ + "CHEAPER_INFERENCE_BASE_URL", + "CheaperInferenceChatModel", + "CheaperInferenceError", +] diff --git a/grapharc/gateway/config.py b/grapharc/gateway/config.py index b74d6e2..6da18d3 100644 --- a/grapharc/gateway/config.py +++ b/grapharc/gateway/config.py @@ -20,7 +20,7 @@ Secrets are returned, never logged. Anything that renders a config for humans goes through `redact`. -Four key-holding backends, plus one that usually holds none: +Five key-holding backends, plus one that usually holds none: - **OpenRouter** — `OPENROUTER_API_KEY`. - **OpenAI** — `OPENAI_API_KEY`, optionally with `OPENAI_BASE_URL` for an @@ -31,6 +31,9 @@ - **Novita** — `NOVITA_API_KEY`. The endpoint is fixed (`grapharc.gateway.novita.NOVITA_BASE_URL`), so unlike OpenAI there is no base-url override to resolve here. +- **Cheaper Inference** — `CHEAPER_INFERENCE_API_KEY`. The endpoint is fixed + (`grapharc.gateway.cheaperinference.CHEAPER_INFERENCE_BASE_URL`), the same + as Novita's. - **Ollama** — no credential by default: it is a server on your own machine. What it needs is an address, so `ollama_base_url()` always returns one (`OLLAMA_HOST` / `OLLAMA_BASE_URL`, else localhost). `OLLAMA_API_KEY` exists @@ -66,6 +69,13 @@ "novita_api_key", ) +CHEAPER_INFERENCE_KEYS = ( + "CHEAPER_INFERENCE_API_KEY", + "CHEAPER_INFERENCE_KEY", + "cheaper-inference-api-key", + "cheaper_inference_api_key", +) + # `OPENAI_API_BASE` is the older spelling and is still what a lot of tooling # sets; both are accepted, the current one first. OPENAI_BASE_URL_KEYS = ( @@ -147,6 +157,10 @@ def novita_api_key(*, env_file: Path | None = None) -> str | None: return get_secret(NOVITA_KEYS, env_file=env_file) +def cheaper_inference_api_key(*, env_file: Path | None = None) -> str | None: + return get_secret(CHEAPER_INFERENCE_KEYS, env_file=env_file) + + def openai_base_url(*, env_file: Path | None = None) -> str | None: """An endpoint override, or None for api.openai.com. diff --git a/grapharc/gateway/registry.py b/grapharc/gateway/registry.py index 3338669..ad7b286 100644 --- a/grapharc/gateway/registry.py +++ b/grapharc/gateway/registry.py @@ -8,6 +8,7 @@ openrouter/openai/gpt-4o:floor -> OpenRouter, cheapest provider openai/gpt-4o-mini -> OpenAI directly (OPENAI_API_KEY) novita/moonshotai/kimi-k3 -> Novita (NOVITA_API_KEY) + cheaperinference/gpt-5.4-mini -> Cheaper Inference (CHEAPER_INFERENCE_API_KEY) ollama/llama3.1 -> a local Ollama server, no key mock/anything -> scripted test double @@ -29,7 +30,7 @@ class UnknownBackendError(Exception): """The spec named a backend that is not registered.""" -BACKENDS = ("claude-cli", "openrouter", "openai", "novita", "ollama", "mock") +BACKENDS = ("claude-cli", "openrouter", "openai", "novita", "cheaperinference", "ollama", "mock") # Authors that appear in OpenRouter model ids. A spec starting with one of # these is a model name, not a mistyped backend — `anthropic/claude-haiku-4.5` @@ -42,7 +43,8 @@ class UnknownBackendError(Exception): #: Which vendor's models a backend serves, for the independence check below. #: OpenRouter is deliberately absent: it serves everyone, so its vendor comes -#: from the author slug in the model id instead. +#: from the author slug in the model id instead. Cheaper Inference is absent +#: too: it serves several labs under bare ids, so it reads as itself. BACKEND_VENDOR = { "claude-cli": "anthropic", "openai": "openai", @@ -73,6 +75,7 @@ class UnknownBackendError(Exception): "openrouter": "openrouter/anthropic/claude-sonnet-4.5", "openai": "openai/gpt-4o-mini", "novita": "novita/moonshotai/kimi-k3", + "cheaperinference": "cheaperinference/gpt-5.4-mini", "ollama": "ollama/llama3.1", } @@ -173,6 +176,17 @@ def get_model(spec: str, **kwargs: Any) -> BaseChatModel: return NovitaChatModel(model, **kwargs) + if backend == "cheaperinference": + try: + from grapharc.gateway.cheaperinference import CheaperInferenceChatModel + except ImportError as exc: # pragma: no cover - depends on install extras + raise UnknownBackendError( + "The Cheaper Inference backend needs langchain-openai. " + "Install it with: uv sync --extra cheaperinference" + ) from exc + + return CheaperInferenceChatModel(model, **kwargs) + if backend == "ollama": try: from grapharc.gateway.ollama import OllamaChatModel diff --git a/pyproject.toml b/pyproject.toml index 9cfd5a7..2863613 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,12 @@ openai = [ novita = [ "langchain-openai>=0.2", ] +# Imported by grapharc/gateway/cheaperinference.py. Cheaper Inference speaks the +# OpenAI wire format against its own endpoint, so this needs the same client and +# no Cheaper Inference-specific package. +cheaperinference = [ + "langchain-openai>=0.2", +] # Imported by grapharc/gateway/ollama.py. Ollama speaks the OpenAI wire format, # so the local backend needs the same client and no ollama-specific package. ollama = [ @@ -121,7 +127,7 @@ slack = [ ] # Everything above. Self-referential so it cannot drift out of sync. all = [ - "grapharc[api,ladybug,mcp,novita,ollama,openai,openrouter,otel,server,slack]", + "grapharc[api,cheaperinference,ladybug,mcp,novita,ollama,openai,openrouter,otel,server,slack]", ] [dependency-groups] diff --git a/tests/test_cli.py b/tests/test_cli.py index 4c3a2b4..53d50df 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -446,6 +446,7 @@ def test_models_check_exits_one_when_nothing_is_configured(monkeypatch, capsys): "OPENROUTER_API_KEY", "OPENROUTER_KEY", "open-router-api-key", "OPENAI_API_KEY", "OPENAI_KEY", "openai-api-key", "NOVITA_API_KEY", "NOVITA_KEY", "novita-api-key", + "CHEAPER_INFERENCE_API_KEY", "CHEAPER_INFERENCE_KEY", "cheaper-inference-api-key", "OLLAMA_HOST", "OLLAMA_BASE_URL", ): monkeypatch.delenv(name, raising=False) @@ -461,10 +462,11 @@ def test_models_check_exits_one_when_nothing_is_configured(monkeypatch, capsys): "openrouter": False, "openai": False, "novita": False, + "cheaperinference": False, "ollama": False, "mock": True, } - for backend in ("openrouter", "openai", "novita"): + for backend in ("openrouter", "openai", "novita", "cheaperinference"): assert next(b for b in payload["backends"] if b["backend"] == backend)[ "credential" ] == "" @@ -503,7 +505,7 @@ def test_models_check_flags_a_backend_it_cannot_probe(monkeypatch, capsys): assert "no probe" in bedrock["detail"] _, out, _ = call(["models", "--check"], capsys) - assert "bedrock unprobed" in out + assert "bedrock unprobed" in out def test_models_check_and_a_spec_are_different_questions(capsys): diff --git a/tests/test_cookbook_models.py b/tests/test_cookbook_models.py index 5ed5551..56dd260 100644 --- a/tests/test_cookbook_models.py +++ b/tests/test_cookbook_models.py @@ -41,6 +41,7 @@ from grapharc.gateway import DEFAULT_RETRY_POLICY, NO_RETRY from grapharc.gateway.config import ( + CHEAPER_INFERENCE_KEYS, NOVITA_KEYS, OLLAMA_BASE_URL_KEYS, OLLAMA_KEYS, @@ -78,6 +79,7 @@ *OPENAI_KEYS, *OPENAI_BASE_URL_KEYS, *NOVITA_KEYS, + *CHEAPER_INFERENCE_KEYS, *OLLAMA_KEYS, *OLLAMA_BASE_URL_KEYS, ) @@ -198,6 +200,7 @@ def _needs_openrouter(body: str) -> bool: "langchain_openai", '"openai/', '"novita/', + '"cheaperinference/', '"ollama/', ) ) @@ -292,9 +295,11 @@ def test_cli_command_with_machine_dependent_output_runs(position, tmp_path): def test_the_backend_list_the_page_prints_is_the_real_one(): page = DOC.read_text(encoding="utf-8") - assert BACKENDS == ("claude-cli", "openrouter", "openai", "novita", "ollama", "mock") + assert BACKENDS == ( + "claude-cli", "openrouter", "openai", "novita", "cheaperinference", "ollama", "mock", + ) assert DEFAULT_BACKEND == "claude-cli" - assert "There are six backends:" in page + assert "There are seven backends:" in page # Every backend has a row in the page's table. A backend added to the # gateway and not to the page fails here rather than going undocumented. for backend in BACKENDS: diff --git a/tests/test_gateway_cheaperinference.py b/tests/test_gateway_cheaperinference.py new file mode 100644 index 0000000..419a24b --- /dev/null +++ b/tests/test_gateway_cheaperinference.py @@ -0,0 +1,220 @@ +"""The Cheaper Inference backend. + +Everything here is offline: constructing a `ChatOpenAI` opens no socket, so the +wiring — credentials, endpoint, cost accounting — is checkable without +spending anything. +""" + +from __future__ import annotations + +import pytest +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, ChatResult + +from grapharc.gateway import config, describe, get_model, split_spec, vendor +from grapharc.gateway.registry import BACKEND_VENDOR, UnknownBackendError + +pytest.importorskip( + "langchain_openai", reason="Cheaper Inference needs the cheaperinference extra" +) + +from grapharc.gateway.cheaperinference import ( # noqa: E402 + CHEAPER_INFERENCE_BASE_URL, + CheaperInferenceChatModel, + CheaperInferenceError, +) + + +@pytest.fixture +def no_credentials(monkeypatch, tmp_path): + """No key in the environment, and a working directory holding no .env.""" + for name in config.CHEAPER_INFERENCE_KEYS: + monkeypatch.delenv(name, raising=False) + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _result(**token_usage) -> ChatResult: + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content="x"))], + llm_output={"model_name": "test-model", "token_usage": token_usage}, + ) + + +# ---------------------------------------------------------------- credentials + + +def test_cheaper_inference_key_is_read_from_a_dotenv_file(tmp_path, monkeypatch): + """`cheaper-inference-api-key` cannot be a shell variable, so the file is parsed.""" + for name in config.CHEAPER_INFERENCE_KEYS: + monkeypatch.delenv(name, raising=False) + env = tmp_path / ".env" + env.write_text('cheaper-inference-api-key="sk-fromfile"\n', encoding="utf-8") + assert config.cheaper_inference_api_key(env_file=env) == "sk-fromfile" + + +def test_cheaper_inference_process_env_beats_the_file(tmp_path, monkeypatch): + env = tmp_path / ".env" + env.write_text("cheaper-inference-api-key=from-file\n", encoding="utf-8") + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "from-env") + assert config.cheaper_inference_api_key(env_file=env) == "from-env" + + +def test_constructing_cheaper_inference_without_a_key_explains_how_to_fix_it(no_credentials): + with pytest.raises(CheaperInferenceError, match="CHEAPER_INFERENCE_API_KEY"): + CheaperInferenceChatModel("gpt-5.4-mini") + + +def test_cheaper_inference_key_never_appears_in_a_description_or_a_redaction(monkeypatch): + secret = "sk-cheaper-0123456789abcdef0123456789abcdef" + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", secret) + assert secret not in str(describe("cheaperinference/gpt-5.4-mini")) + assert secret not in config.redact(secret) + + +def test_cheaper_inference_always_points_at_its_own_endpoint(monkeypatch): + """Unlike OpenAI, there is no override — the endpoint is fixed.""" + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "sk-test") + model = CheaperInferenceChatModel("gpt-5.4-mini") + assert str(model.openai_api_base) == CHEAPER_INFERENCE_BASE_URL + assert CHEAPER_INFERENCE_BASE_URL == "https://api.cheaperinference.com/v1" + + +# ------------------------------------------------------------------- registry + + +@pytest.mark.parametrize( + ("spec", "backend", "model"), + [ + ("cheaperinference/gpt-5.4-mini", "cheaperinference", "gpt-5.4-mini"), + ("cheaperinference/claude-sonnet-5", "cheaperinference", "claude-sonnet-5"), + ], +) +def test_cheaper_inference_specs_split_the_way_the_docs_say(spec, backend, model): + assert split_spec(spec) == (backend, model) + assert describe(spec) == {"spec": spec, "backend": backend, "model": model} + + +def test_the_registry_builds_the_cheaper_inference_backend(monkeypatch): + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "sk-test") + model = get_model("cheaperinference/gpt-5.4-mini") + assert model._llm_type == "grapharc-cheaperinference" + + +def test_a_missing_cheaper_inference_key_surfaces_through_the_registry(no_credentials): + with pytest.raises(CheaperInferenceError): + get_model("cheaperinference/gpt-5.4-mini") + + +def test_an_unknown_backend_still_names_cheaper_inference(): + with pytest.raises(UnknownBackendError, match="cheaperinference"): + get_model("cheaperinferense/gpt-5.4-mini") + + +def test_bare_cheaper_inference_ids_read_as_the_gateway_not_a_lab(): + """Model ids here carry no `author/` prefix, so `vendor()` has nothing to + read but the backend. It is not mapped to one lab in `BACKEND_VENDOR`, + because the gateway serves several.""" + assert vendor("cheaperinference/gpt-5.4-mini") == "cheaperinference" + assert vendor("cheaperinference/claude-sonnet-5") == "cheaperinference" + assert "cheaperinference" not in BACKEND_VENDOR + + +# ------------------------------------------------- capabilities and accounting + + +def test_cheaper_inference_can_bind_tools_and_structure_output(monkeypatch): + """The capability the Claude-CLI backend cannot offer.""" + from langchain_core.tools import tool + from pydantic import BaseModel + + @tool + def get_weather(city: str) -> str: + """Get the current weather for a city.""" + return f"sunny in {city}" + + class Verdict(BaseModel): + supported: bool + + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "sk-test") + model = CheaperInferenceChatModel("gpt-5.4-mini") + model.bind_tools([get_weather]) + model.with_structured_output(Verdict) + + +def test_the_usage_envelope_matches_every_other_backend(monkeypatch): + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "sk-test") + model = CheaperInferenceChatModel("gpt-5.4-mini") + model._record_usage( + _result( + prompt_tokens=1000, + completion_tokens=50, + prompt_tokens_details={"cached_tokens": 800}, + ) + ) + usage = model.last_usage + assert usage["input_tokens"] == 1000 # cached input still counts + assert usage["total_tokens"] == 1050 + assert usage["input_token_details"]["cache_read"] == 800 + assert usage["uncached_input_tokens"] == 200 + + +def test_cheaper_inference_reads_no_cost_and_says_so_rather_than_guessing(monkeypatch): + """Like the OpenAI backend, this one reads tokens and no price. An + invented number would be worse than an admitted gap, so the call is + counted as unpriced.""" + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "sk-test") + model = CheaperInferenceChatModel("gpt-5.4-mini") + model._settle(_result(prompt_tokens=1000, completion_tokens=50)) + assert model.last_usage["cost_usd"] is None + assert model.spend.unpriced_calls == 1 + assert model.spend.spent_usd == 0.0 + + +def test_a_rate_card_prices_cheaper_inference_the_way_it_prices_openai(monkeypatch): + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "sk-test") + model = CheaperInferenceChatModel( + "gpt-5.4-mini", + price_per_million={"input": 0.15, "cached_input": 0.075, "output": 0.60}, + ) + model._settle( + _result( + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + prompt_tokens_details={"cached_tokens": 400_000}, + ) + ) + # 600k uncached @ 0.15 + 400k cached @ 0.075 + 1M output @ 0.60 + assert model.last_usage["cost_usd"] == pytest.approx(0.09 + 0.03 + 0.60) + assert model.spend.unpriced_calls == 0 + + +def test_cheaper_inference_sets_no_max_tokens_ceiling_of_its_own(monkeypatch): + """OpenRouter defaults it to dodge a credit-reservation 402. This backend, + like Novita and OpenAI, sets no default, which would only truncate replies.""" + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "sk-test") + assert CheaperInferenceChatModel("gpt-5.4-mini").max_tokens is None + + +# ------------------------------------------------------------------ live ---- + + +@pytest.mark.live +@pytest.mark.skipif( + not config.cheaper_inference_api_key(), reason="no Cheaper Inference API key configured" +) +def test_live_cheaper_inference_tool_calling(): + from langchain_core.messages import HumanMessage + from langchain_core.tools import tool + + @tool + def get_weather(city: str) -> str: + """Get the current weather for a city.""" + return f"sunny in {city}" + + model = get_model("cheaperinference/gpt-5.4-mini") + reply = model.bind_tools([get_weather]).invoke( + [HumanMessage(content="What is the weather in Paris? Use the tool.")] + ) + assert reply.tool_calls + assert reply.tool_calls[0]["name"] == "get_weather" diff --git a/tests/test_gateway_openai_ollama.py b/tests/test_gateway_openai_ollama.py index 82fd5fb..57cc44a 100644 --- a/tests/test_gateway_openai_ollama.py +++ b/tests/test_gateway_openai_ollama.py @@ -175,7 +175,7 @@ def test_a_missing_openai_key_surfaces_through_the_registry(no_credentials): def test_an_unknown_backend_still_names_every_real_one(): - with pytest.raises(UnknownBackendError, match="openai, novita, ollama"): + with pytest.raises(UnknownBackendError, match="openai, novita, cheaperinference, ollama"): get_model("opnai/gpt-4o-mini") diff --git a/tests/test_gateway_openrouter.py b/tests/test_gateway_openrouter.py index 5d4a54c..eb70ed7 100644 --- a/tests/test_gateway_openrouter.py +++ b/tests/test_gateway_openrouter.py @@ -149,7 +149,8 @@ def test_a_typoed_backend_is_caught_early_not_folded_into_a_model_name(): """`opnerouter/...` must fail here, not become a Claude-CLI call with a nonsense model that errors confusingly much later.""" with pytest.raises( - UnknownBackendError, match="claude-cli, openrouter, openai, novita, ollama, mock" + UnknownBackendError, + match="claude-cli, openrouter, openai, novita, cheaperinference, ollama, mock", ): split_spec("opnerouter/anthropic/claude-haiku-4.5") with pytest.raises(UnknownBackendError): diff --git a/uv.lock b/uv.lock index bf61f30..90fbed5 100644 --- a/uv.lock +++ b/uv.lock @@ -375,6 +375,9 @@ all = [ api = [ { name = "anthropic" }, ] +cheaperinference = [ + { name = "langchain-openai" }, +] ladybug = [ { name = "real-ladybug" }, ] @@ -420,8 +423,9 @@ dev = [ requires-dist = [ { name = "anthropic", marker = "extra == 'api'", specifier = ">=0.40" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115" }, - { name = "grapharc", extras = ["api", "ladybug", "mcp", "novita", "ollama", "openai", "openrouter", "otel", "server", "slack"], marker = "extra == 'all'" }, + { name = "grapharc", extras = ["api", "cheaperinference", "ladybug", "mcp", "novita", "ollama", "openai", "openrouter", "otel", "server", "slack"], marker = "extra == 'all'" }, { name = "langchain-core", specifier = ">=0.3" }, + { name = "langchain-openai", marker = "extra == 'cheaperinference'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'novita'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'ollama'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'openai'", specifier = ">=0.2" }, @@ -437,7 +441,7 @@ requires-dist = [ { name = "slack-bolt", marker = "extra == 'slack'", specifier = ">=1.20" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.32" }, ] -provides-extras = ["openrouter", "openai", "novita", "ollama", "server", "mcp", "ladybug", "otel", "api", "slack", "all"] +provides-extras = ["openrouter", "openai", "novita", "cheaperinference", "ollama", "server", "mcp", "ladybug", "otel", "api", "slack", "all"] [package.metadata.requires-dev] dev = [