From 4ebb67101c118fa83b744ff9db7f4ccee61c2623 Mon Sep 17 00:00:00 2001 From: Supernova Date: Wed, 14 Oct 2026 21:54:58 +0000 Subject: [PATCH 1/2] Keep hand-verified Bedrock Claude prices through a registry refresh rake models:update merges the provider listings with models.dev and writes the result over models.json. Hand edits did not survive that: models.dev would replace the deliberate global rate on us. Haiku 4.5, and a 'manual' entry that no source returned was dropped. Add lib/ruby_llm/models_overrides.json. RubyLLM::Models.apply_overrides re-applies each entry after the merge, replacing pricing.text_tokens.standard and metadata.cost. Refresh also keeps 'manual' entries that no source returns. The overrides hold the us. Claude ids Supernova bills, plus global. Fable 5.1, at the AWS Price List rates (AmazonBedrockFoundationModels, us-west-2, publication 2026-10-12). They equal the current models.json values, so models.json is unchanged. No long-context tier is added. Anthropic's pricing page (read 2026-10-14) says Claude 4.6 and later models, except Haiku 5.5, bill the full 1M window at standard pricing. The AWS Price List has long_ctx SKUs only for Haiku 5.5 and Sonnet 4. The new spec records this for the 1M-window ids and fails when models.json and the overrides disagree. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Sam Boland --- lib/ruby_llm/models.rb | 48 ++++++++- lib/ruby_llm/models_overrides.json | 136 +++++++++++++++++++++++++ spec/ruby_llm/models_overrides_spec.rb | 106 +++++++++++++++++++ 3 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 lib/ruby_llm/models_overrides.json create mode 100644 spec/ruby_llm/models_overrides_spec.rb diff --git a/lib/ruby_llm/models.rb b/lib/ruby_llm/models.rb index 85087addf..6ea2b40c1 100644 --- a/lib/ruby_llm/models.rb +++ b/lib/ruby_llm/models.rb @@ -69,6 +69,30 @@ def schema_file File.expand_path('models_schema.json', __dir__) end + def overrides_file + File.expand_path('models_overrides.json', __dir__) + end + + # Hand-verified prices keyed "provider:id", re-applied on top of every refresh so a + # models.dev regeneration cannot silently replace them (see models_overrides.json). + def read_overrides(file = overrides_file) + return {} unless File.exist?(file) + + JSON.parse(File.read(file), symbolize_names: true).fetch(:overrides, {}).transform_keys(&:to_s) + end + + # Replaces each overridden model's standard text pricing and metadata.cost with the + # override's cost. metadata.cost is replaced whole, so a tier models.dev adds for an + # overridden id is dropped too: the override is the authority for that id. + def apply_overrides(models, overrides = read_overrides) + return models if overrides.empty? + + models.map do |model| + override = overrides["#{model.provider}:#{model.id}"] + override ? apply_override(model, override[:cost]) : model + end + end + def load_models(file = RubyLLM.config.model_registry_file) source = RubyLLM.config.model_registry_source if source && file == RubyLLM.config.model_registry_file @@ -109,6 +133,7 @@ def fetch_merged_models(remote_only: false) log_models_dev_fetch(models_dev_fetch) merged_models = merge_with_existing(existing_models, provider_fetch, models_dev_fetch) + merged_models = apply_overrides(merged_models) payload[:model_count] = merged_models.size merged_models end @@ -218,7 +243,28 @@ def merge_with_existing(existing_models, provider_fetch, models_dev_fetch) existing_models.select { |model| model.metadata[:source] == 'models.dev' } end - merge_models(provider_models, models_dev_models) + keep_manual_models(merge_models(provider_models, models_dev_models), existing_models) + end + + # A model with metadata source 'manual' was added by hand. Keep it when neither the + # provider listing nor models.dev returns it, instead of dropping it from the registry. + def keep_manual_models(merged_models, existing_models) + merged_keys = index_by_key(merged_models) + missing = existing_models.select do |model| + model.metadata[:source] == 'manual' && !merged_keys.key?("#{model.provider}:#{model.id}") + end + return merged_models if missing.empty? + + (merged_models + missing).sort_by { |m| [m.provider, m.id] } + end + + def apply_override(model, cost) + data = model.to_h + pricing = data[:pricing] || {} + text_tokens = (pricing[:text_tokens] || {}).merge(models_dev_pricing(cost)[:text_tokens]) + data[:pricing] = pricing.merge(text_tokens: text_tokens) + data[:metadata] = (data[:metadata] || {}).merge(cost: cost) + Model::Info.new(data) end def merge_models(provider_models, models_dev_models) diff --git a/lib/ruby_llm/models_overrides.json b/lib/ruby_llm/models_overrides.json new file mode 100644 index 000000000..71c9ad2a9 --- /dev/null +++ b/lib/ruby_llm/models_overrides.json @@ -0,0 +1,136 @@ +{ + "description": [ + "Prices verified by hand against the vendor. rake models:update re-applies every entry here after it merges the provider listings with models.dev (RubyLLM::Models.apply_overrides), so a regeneration cannot replace them.", + "Each entry replaces the model's pricing.text_tokens.standard and metadata.cost. metadata.cost is replaced whole, so no long-context tier is kept for these ids.", + "No long-context tier exists for these models. Anthropic's pricing page (platform.claude.com/docs/en/about-claude/pricing, read 2026-10-14) says: 'Claude 4.6 and later models (except Claude Haiku 5.5) ... include the full 1M token context window at standard pricing.' The AWS Price List has long_ctx SKUs only for Claude Haiku 5.5 and Claude Sonnet 4.", + "Edit models.json and this file together. spec/ruby_llm/models_overrides_spec.rb fails when they disagree." + ], + "overrides": { + "bedrock:us.anthropic.claude-fable-5-1": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 11, + "output": 55, + "cache_read": 0.275, + "cache_write": 13.75 + } + }, + "bedrock:us.anthropic.claude-fable-5": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 11, + "output": 55, + "cache_read": 1.1, + "cache_write": 13.75 + } + }, + "bedrock:us.anthropic.claude-sonnet-5-5": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 2.2, + "output": 11, + "cache_read": 0.11, + "cache_write": 2.75 + } + }, + "bedrock:us.anthropic.claude-sonnet-5": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 2.2, + "output": 11, + "cache_read": 0.22, + "cache_write": 2.75 + } + }, + "bedrock:us.anthropic.claude-opus-5-5": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 4.4, + "output": 22, + "cache_read": 0.22, + "cache_write": 5.5 + } + }, + "bedrock:us.anthropic.claude-opus-5": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 5.5, + "output": 27.5, + "cache_read": 0.55, + "cache_write": 6.875 + } + }, + "bedrock:us.anthropic.claude-opus-4-8": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 5.5, + "output": 27.5, + "cache_read": 0.55, + "cache_write": 6.875 + } + }, + "bedrock:us.anthropic.claude-opus-4-7": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 5.5, + "output": 27.5, + "cache_read": 0.55, + "cache_write": 6.875 + } + }, + "bedrock:us.anthropic.claude-opus-4-6-v1": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 5.5, + "output": 27.5, + "cache_read": 0.55, + "cache_write": 6.875 + } + }, + "bedrock:us.anthropic.claude-opus-4-5-20251101-v1:0": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 5.5, + "output": 27.5, + "cache_read": 0.55, + "cache_write": 6.875 + } + }, + "bedrock:us.anthropic.claude-sonnet-4-6": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 3.3, + "output": 16.5, + "cache_read": 0.33, + "cache_write": 4.125 + } + }, + "bedrock:us.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand regional ('Standard' / 'Regional') SKUs", + "cost": { + "input": 3.3, + "output": 16.5, + "cache_read": 0.33, + "cache_write": 4.125 + } + }, + "bedrock:us.anthropic.claude-haiku-4-5-20251001-v1:0": { + "source": "Global rate on purpose: the AWS bill for 2026-10-05 to 2026-10-12 shows us. Haiku 4.5 at 94% of its regional rate (reason unknown), and the global rate (AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand 'Standard, Global' SKUs) is closer to the bill.", + "cost": { + "input": 1, + "output": 5, + "cache_read": 0.1, + "cache_write": 1.25 + } + }, + "bedrock:global.anthropic.claude-fable-5-1": { + "source": "AWS Price List API, AmazonBedrockFoundationModels offer, us-west-2, publication 2026-10-12: on-demand 'Standard, Global' SKUs", + "cost": { + "input": 10, + "output": 50, + "cache_read": 0.25, + "cache_write": 12.5 + } + } + } +} diff --git a/spec/ruby_llm/models_overrides_spec.rb b/spec/ruby_llm/models_overrides_spec.rb new file mode 100644 index 000000000..d9f590dcb --- /dev/null +++ b/spec/ruby_llm/models_overrides_spec.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe RubyLLM::Models do + include_context 'with configured RubyLLM' + + overrides = described_class.read_overrides + + def bedrock_entry(id) + described_class.read_from_json.find { |m| m.provider == 'bedrock' && m.id == id } + end + + def standard_pricing(model) + model.pricing.text_tokens.standard.to_h.slice( + :input_per_million, :output_per_million, + :cache_read_input_per_million, :cache_write_input_per_million + ) + end + + def expected_standard(cost) + { + input_per_million: cost[:input], + output_per_million: cost[:output], + cache_read_input_per_million: cost[:cache_read], + cache_write_input_per_million: cost[:cache_write] + } + end + + it 'has overrides to check' do + expect(overrides).not_to be_empty + end + + # models.json must already hold each override, so the JSON file and a regenerated registry + # agree. A hand edit to one file without the other fails here. + overrides.each do |key, override| + provider, id = key.split(':', 2) + + it "keeps #{key} in models.json at the override price" do + model = described_class.read_from_json.find { |m| m.provider == provider && m.id == id } + + expect(model).not_to be_nil + expect(model.metadata[:cost]).to eq(override[:cost]) + expect(standard_pricing(model)).to eq(expected_standard(override[:cost])) + end + end + + # Anthropic's pricing page (platform.claude.com/docs/en/about-claude/pricing, read + # 2026-10-14), "Long context pricing": "Claude 4.6 and later models (except Claude Haiku 5.5) + # ... include the full 1M token context window at standard pricing." The AWS Price List + # (AmazonBedrockFoundationModels, us-west-2, publication 2026-10-12) has long_ctx SKUs only + # for Claude Haiku 5.5 and Claude Sonnet 4. A tier on any of these ids would overcharge every + # request over 200k tokens. + %w[ + us.anthropic.claude-opus-4-8 us.anthropic.claude-opus-5 us.anthropic.claude-opus-5-5 + us.anthropic.claude-sonnet-5 us.anthropic.claude-sonnet-5-5 + us.anthropic.claude-fable-5 us.anthropic.claude-fable-5-1 + ].each do |id| + it "prices #{id} with no long-context tier" do + model = bedrock_entry(id) + + expect(model.context_window).to eq(1_000_000) + expect(model.metadata[:cost].keys).to contain_exactly(:input, :output, :cache_read, :cache_write) + expect(model.pricing.to_h[:text_tokens].keys).to eq([:standard]) + end + end + + describe 'refresh' do + let(:overridden_id) { 'us.anthropic.claude-opus-5-5' } + let(:models_dev_model) do + RubyLLM::Model::Info.new( + id: overridden_id, name: 'Claude Opus 5.5 (US)', provider: 'bedrock', + pricing: { text_tokens: { standard: { input_per_million: 4, output_per_million: 20 } } }, + metadata: { + source: 'models.dev', + cost: { input: 4, output: 20, tiers: [{ input: 8, output: 30, tier: { type: 'context', size: 200_000 } }] } + } + ) + end + let(:manual_model) do + RubyLLM::Model::Info.new(id: 'us.anthropic.claude-hand-added', name: 'Hand added', provider: 'bedrock', + metadata: { source: 'manual', cost: { input: 1, output: 2 } }) + end + + before do + allow(described_class).to receive(:read_existing_models).and_return([manual_model]) + allow(described_class).to receive_messages( + fetch_provider_models: { models: [], fetched_providers: ['bedrock'], configured_names: [], failed: [] }, + fetch_models_dev_models: { models: [models_dev_model], fetched: true } + ) + end + + it 'applies the override over the models.dev price, tiers included' do + model = described_class.fetch_merged_models.find { |m| m.id == overridden_id } + + expect(model.metadata[:cost]).to eq(overrides["bedrock:#{overridden_id}"][:cost]) + expect(standard_pricing(model)).to eq(expected_standard(overrides["bedrock:#{overridden_id}"][:cost])) + end + + it 'keeps a manual model that no source returns' do + ids = described_class.fetch_merged_models.map(&:id) + + expect(ids).to include('us.anthropic.claude-hand-added') + end + end +end From 873deb6f1273bd2bb98c23bdf6f56e1511761fcb Mon Sep 17 00:00:00 2001 From: Supernova Date: Wed, 14 Oct 2026 22:41:07 +0000 Subject: [PATCH 2/2] Keep only the standard text tier on an overridden model; document manual entries apply_override merged its standard rates into the existing text_tokens, so any other text tier stayed in pricing. It now sets text_tokens to { standard: ... } only. Model::Pricing already keeps only standard and batch, so a long-context tier reaches Supernova only through metadata.cost, which the override replaces whole. The new spec pins both. Also document that keep_manual_models keeps a 'manual' entry until someone deletes it from models.json. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Sam Boland --- lib/ruby_llm/models.rb | 10 ++++++--- lib/ruby_llm/models_overrides.json | 3 ++- spec/ruby_llm/models_overrides_spec.rb | 30 ++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/lib/ruby_llm/models.rb b/lib/ruby_llm/models.rb index 6ea2b40c1..8ecf518e9 100644 --- a/lib/ruby_llm/models.rb +++ b/lib/ruby_llm/models.rb @@ -248,6 +248,7 @@ def merge_with_existing(existing_models, provider_fetch, models_dev_fetch) # A model with metadata source 'manual' was added by hand. Keep it when neither the # provider listing nor models.dev returns it, instead of dropping it from the registry. + # A manual model therefore stays until someone deletes it from models.json by hand. def keep_manual_models(merged_models, existing_models) merged_keys = index_by_key(merged_models) missing = existing_models.select do |model| @@ -258,11 +259,14 @@ def keep_manual_models(merged_models, existing_models) (merged_models + missing).sort_by { |m| [m.provider, m.id] } end + # text_tokens becomes { standard: ... } only, so no other text tier survives for an + # overridden id (a batch tier is dropped too). Model::Pricing keeps only standard and + # batch, so a long-context tier can only arrive through metadata.cost (tiers[] or + # context_over_200k, which Supernova's Llm::PricingRegistry reads); replacing + # metadata.cost whole is what removes it. def apply_override(model, cost) data = model.to_h - pricing = data[:pricing] || {} - text_tokens = (pricing[:text_tokens] || {}).merge(models_dev_pricing(cost)[:text_tokens]) - data[:pricing] = pricing.merge(text_tokens: text_tokens) + data[:pricing] = (data[:pricing] || {}).merge(text_tokens: models_dev_pricing(cost)[:text_tokens]) data[:metadata] = (data[:metadata] || {}).merge(cost: cost) Model::Info.new(data) end diff --git a/lib/ruby_llm/models_overrides.json b/lib/ruby_llm/models_overrides.json index 71c9ad2a9..66f654397 100644 --- a/lib/ruby_llm/models_overrides.json +++ b/lib/ruby_llm/models_overrides.json @@ -1,7 +1,8 @@ { "description": [ "Prices verified by hand against the vendor. rake models:update re-applies every entry here after it merges the provider listings with models.dev (RubyLLM::Models.apply_overrides), so a regeneration cannot replace them.", - "Each entry replaces the model's pricing.text_tokens.standard and metadata.cost. metadata.cost is replaced whole, so no long-context tier is kept for these ids.", + "Each entry sets the model's pricing.text_tokens to { standard: ... } only and replaces metadata.cost whole, so no long-context tier is kept for these ids in either place.", + "rake models:update also keeps every models.json entry with metadata source 'manual' that no provider listing or models.dev returns (RubyLLM::Models.keep_manual_models). Such an entry stays until it is deleted from models.json by hand.", "No long-context tier exists for these models. Anthropic's pricing page (platform.claude.com/docs/en/about-claude/pricing, read 2026-10-14) says: 'Claude 4.6 and later models (except Claude Haiku 5.5) ... include the full 1M token context window at standard pricing.' The AWS Price List has long_ctx SKUs only for Claude Haiku 5.5 and Claude Sonnet 4.", "Edit models.json and this file together. spec/ruby_llm/models_overrides_spec.rb fails when they disagree." ], diff --git a/spec/ruby_llm/models_overrides_spec.rb b/spec/ruby_llm/models_overrides_spec.rb index d9f590dcb..23cea2159 100644 --- a/spec/ruby_llm/models_overrides_spec.rb +++ b/spec/ruby_llm/models_overrides_spec.rb @@ -65,6 +65,36 @@ def expected_standard(cost) end end + describe '.apply_overrides' do + let(:cost) { { input: 4.4, output: 22, cache_read: 0.22, cache_write: 5.5 } } + let(:tiered_model) do + RubyLLM::Model::Info.new( + id: 'us.anthropic.claude-opus-5-5', name: 'Claude Opus 5.5 (US)', provider: 'bedrock', + pricing: { + text_tokens: { + standard: { input_per_million: 4, output_per_million: 20 }, + context_over_200k: { input_per_million: 8, output_per_million: 30 } + } + }, + metadata: { + cost: { input: 4, output: 20, context_over_200k: { input: 8, output: 30 }, + tiers: [{ input: 8, output: 30, tier: { type: 'context', size: 200_000 } }] } + } + ) + end + + # Model::Pricing already drops any text tier but standard and batch, so the tier that + # matters is the one in metadata.cost; this pins both places. + it 'drops a long-context tier from both pricing and metadata.cost' do + model = described_class.apply_overrides([tiered_model], + { 'bedrock:us.anthropic.claude-opus-5-5' => { cost: cost } }) + .first + + expect(model.pricing.to_h[:text_tokens]).to eq(standard: expected_standard(cost)) + expect(model.metadata[:cost]).to eq(cost) + end + end + describe 'refresh' do let(:overridden_id) { 'us.anthropic.claude-opus-5-5' } let(:models_dev_model) do