diff --git a/app/src/components/settings/panels/AIPanel.tsx b/app/src/components/settings/panels/AIPanel.tsx index 126466a774c..455759db31f 100644 --- a/app/src/components/settings/panels/AIPanel.tsx +++ b/app/src/components/settings/panels/AIPanel.tsx @@ -36,6 +36,7 @@ import { BackgroundLoopControls } from './ai/BackgroundLoopControls'; import { CloudProviderEditor } from './ai/CloudProviderEditor'; import { CustomRoutingDialog } from './ai/CustomRoutingDialog'; import { DefaultModelRow } from './ai/DefaultModelRow'; +import { ModelFeasibilityPreview } from './ai/ModelFeasibilityPreview'; import { ProviderAuthSection } from './ai/ProviderAuthSection'; import { ProviderKeyDialog } from './ai/ProviderConnectControls'; import { SaveBar } from './ai/SaveBar'; @@ -302,6 +303,16 @@ const AIPanel = ({ /> ))} + {!embedded && ( +
+ + {t('settings.ai.feasibility.title', 'Model suitability and cost preview')} + +
+ +
+
+ )} {isDirty && ( { + it('accounts for cached tokens as a subset of estimated input, without modifying routing', () => { + render(); + fireEvent.change(screen.getByLabelText('Estimated input tokens'), { + target: { value: '1000' }, + }); + fireEvent.change(screen.getByLabelText('Cached input tokens (within input)'), { + target: { value: '800' }, + }); + fireEvent.change(screen.getByLabelText('Estimated output tokens'), { + target: { value: '100' }, + }); + expect(screen.getByText('Est. $0.0048')).toBeTruthy(); + expect(screen.getByText(/Matches saved metadata; availability unverified/)).toBeTruthy(); + }); + + it('requires cached input to be no greater than total input tokens', () => { + render(); + fireEvent.change(screen.getByLabelText('Estimated input tokens'), { + target: { value: '1000' }, + }); + fireEvent.change(screen.getByLabelText('Cached input tokens (within input)'), { + target: { value: '1001' }, + }); + expect(screen.getByRole('alert').textContent).toContain( + 'Cached input must be within total input tokens' + ); + }); +}); + +describe('ModelFeasibilityPreview catalog browsing', () => { + const manyModels: ModelRegistryEntry[] = Array.from({ length: 17 }, (_, index) => ({ + ...cachedModel, + id: 'candidate-' + index, + })); + + it('reveals additional sorted results on demand, not only the first eight', () => { + render(); + expect(screen.queryByText('demo/candidate-16')).toBeNull(); + expect(screen.queryByText('demo/candidate-9')).toBeNull(); + fireEvent.click(screen.getByRole('button', { name: 'Show more models' })); + expect(screen.getByText('demo/candidate-16')).toBeTruthy(); + expect(screen.queryByText('demo/candidate-9')).toBeNull(); + fireEvent.click(screen.getByRole('button', { name: 'Show more models' })); + expect(screen.getByText('demo/candidate-9')).toBeTruthy(); + expect(screen.queryByRole('button', { name: 'Show more models' })).toBeNull(); + }); + + it('filters the full catalog by model identity and selected fit status', () => { + render(); + fireEvent.change(screen.getByRole('searchbox', { name: 'Search suitability results' }), { + target: { value: 'candidate-16' }, + }); + expect(screen.getByText('demo/candidate-16')).toBeTruthy(); + expect(screen.queryByText('demo/candidate-1')).toBeNull(); + fireEvent.change(screen.getByRole('combobox', { name: 'Filter suitability status' }), { + target: { value: 'excluded' }, + }); + expect(screen.getByText('No models match these filters.')).toBeTruthy(); + fireEvent.change(screen.getByRole('combobox', { name: 'Filter suitability status' }), { + target: { value: 'metadata_fit' }, + }); + expect(screen.getByText('demo/candidate-16')).toBeTruthy(); + }); +}); diff --git a/app/src/components/settings/panels/ai/ModelFeasibilityPreview.tsx b/app/src/components/settings/panels/ai/ModelFeasibilityPreview.tsx new file mode 100644 index 00000000000..e0729532025 --- /dev/null +++ b/app/src/components/settings/panels/ai/ModelFeasibilityPreview.tsx @@ -0,0 +1,269 @@ +/** Read-only cost/capability estimator. Never changes model routing or secrets. */ +import { useMemo, useState } from 'react'; + +import { useT } from '../../../../lib/i18n/I18nContext'; +import type { ModelRegistryEntry } from '../../../../utils/tauriCommands/config'; +import { + assessModelFeasibility, + formatEstimatedModelCost, + type ModelFitReason, + type ModelFitStatus, +} from './modelFeasibility'; + +const PAGE_SIZE = 8; + +const reasonLabels: Record = { + context_too_small: [ + 'settings.ai.feasibility.contextTooSmall', + 'Context smaller than input + output tokens', + ], + vision_unavailable: [ + 'settings.ai.feasibility.visionUnavailable', + 'Vision unsupported in saved metadata', + ], + estimated_cost_exceeds_budget: [ + 'settings.ai.feasibility.overBudget', + 'Estimated cost exceeds budget', + ], + unknown_context: ['settings.ai.feasibility.unknownContext', 'Context size unknown'], + unknown_vision: ['settings.ai.feasibility.unknownVision', 'Vision capability unknown'], + unknown_price: ['settings.ai.feasibility.unknownPrice', 'Price missing, zero or invalid'], +}; + +interface Props { + models: readonly ModelRegistryEntry[]; +} + +export function ModelFeasibilityPreview({ models }: Props) { + const { t } = useT(); + const [inputText, setInputText] = useState('32000'); + const [cachedInputText, setCachedInputText] = useState('0'); + const [outputText, setOutputText] = useState('2000'); + const [budgetText, setBudgetText] = useState(''); + const [requiresVision, setRequiresVision] = useState(false); + const [query, setQuery] = useState(''); + const [fitStatus, setFitStatus] = useState<'all' | ModelFitStatus>('all'); + const [visibleCount, setVisibleCount] = useState(PAGE_SIZE); + + const report = useMemo(() => { + if (!inputText.trim() || !outputText.trim()) return null; + try { + return assessModelFeasibility(models, { + inputTokens: Number(inputText), + cachedInputTokens: cachedInputText.trim() === '' ? 0 : Number(cachedInputText), + outputTokens: Number(outputText), + requiresVision, + maxEstimatedCostUSD: budgetText.trim() === '' ? null : Number(budgetText), + }); + } catch { + return null; + } + }, [models, inputText, cachedInputText, outputText, budgetText, requiresVision]); + + const matching = useMemo(() => { + if (!report) return []; + const needle = query.trim().toLocaleLowerCase(); + return report.rows.filter( + row => + (fitStatus === 'all' || row.status === fitStatus) && + (!needle || (row.model.provider + '/' + row.model.id).toLocaleLowerCase().includes(needle)) + ); + }, [report, query, fitStatus]); + + const displayed = matching.slice(0, visibleCount); + const cachedExceedsInput = + inputText.trim() !== '' && + cachedInputText.trim() !== '' && + Number(cachedInputText) > Number(inputText); + + return ( +
+
+

+ {t('settings.ai.feasibility.title', 'Model suitability and cost preview')} +

+

+ {t( + 'settings.ai.feasibility.advisory', + 'Based on saved metadata, not verified live prices, quotas, keys or availability. This is advisory only and does not change any workload route.' + )} +

+
+
+ + + + + +
+ {!report ? ( +

+ {cachedExceedsInput + ? t( + 'settings.ai.feasibility.invalidCached', + 'Cached input must be within total input tokens.' + ) + : t( + 'settings.ai.feasibility.invalidCounts', + 'Enter non-negative whole token counts (cached input is a subset) and an optional non-negative cost limit.' + )} +

+ ) : ( + <> +

+ {t('settings.ai.feasibility.fitCount', 'Saved-metadata fit')}:{' '} + {report.counts.metadata_fit}; {t('settings.ai.feasibility.unknownCount', 'unknown')}:{' '} + {report.counts.unknown};{' '} + {t('settings.ai.feasibility.excludedCount', 'excluded by supplied constraints')}:{' '} + {report.counts.excluded}. +

+
+ + +
+
+ {displayed.map(({ model, estimatedCostUSD, reasons, status }) => ( +
+
+
+ {model.provider}/{model.id} +
+
+ {status === 'metadata_fit' + ? t( + 'settings.ai.feasibility.fitDescription', + 'Matches saved metadata; availability unverified' + ) + : status === 'unknown' + ? t('settings.ai.feasibility.unknown', 'Unknown eligibility') + : t( + 'settings.ai.feasibility.excludedDescription', + 'Excluded by constraints' + )} + {reasons.length > 0 && + ': ' + reasons.map(reason => t(...reasonLabels[reason])).join('; ')} +
+
+
+ {estimatedCostUSD === null + ? t('settings.ai.feasibility.costUnknown', 'Cost unknown') + : formatEstimatedModelCost(estimatedCostUSD)} +
+
+ ))} + {matching.length === 0 && ( +

+ {report.rows.length === 0 + ? t('settings.ai.feasibility.empty', 'No saved models to evaluate.') + : t('settings.ai.feasibility.noMatch', 'No models match these filters.')} +

+ )} +
+

+ {t('settings.ai.feasibility.showing', 'Showing')} {displayed.length}{' '} + {t('settings.ai.feasibility.of', 'of')} {matching.length}{' '} + {t('settings.ai.feasibility.matching', 'matching model entries')}. +

+ {displayed.length < matching.length && ( + + )} + + )} +
+ ); +} diff --git a/app/src/components/settings/panels/ai/modelFeasibility.test.ts b/app/src/components/settings/panels/ai/modelFeasibility.test.ts new file mode 100644 index 00000000000..4ae037acdfa --- /dev/null +++ b/app/src/components/settings/panels/ai/modelFeasibility.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it } from 'vitest'; + +import { + assessModelFeasibility, + type FeasibilityModel, + formatEstimatedModelCost, + type ModelRequirements, +} from './modelFeasibility'; + +const model = (id: string, fields: Partial = {}): FeasibilityModel => ({ + provider: 'demo', + id, + context_window: 64_000, + vision: false, + cost_per_1m_input: 1, + cost_per_1m_output: 4, + ...fields, +}); +const req = (fields: Partial = {}): ModelRequirements => ({ + inputTokens: 1000, + outputTokens: 1000, + requiresVision: false, + maxEstimatedCostUSD: null, + ...fields, +}); + +describe('saved-model suitability preview (advisory, no live routing)', () => { + it('uses hard context and vision constraints before any price ranking', () => { + const rows = assessModelFeasibility( + [ + model('a-short', { context_window: 1500, vision: true }), + model('b-blind', { vision: false }), + model('c-fit', { vision: true }), + ], + req({ requiresVision: true }) + ).rows; + expect(rows.map(row => row.model.id)).toEqual(['c-fit', 'a-short', 'b-blind']); + expect(rows.map(row => row.status)).toEqual(['metadata_fit', 'excluded', 'excluded']); + }); + + it('does not mistake missing, invalid or zero prices for verified free use', () => { + const rows = assessModelFeasibility( + [ + model('missing', { cost_per_1m_output: undefined }), + model('zero', { cost_per_1m_input: 0 }), + model('invalid', { cost_per_1m_input: NaN }), + model('priced'), + ], + req() + ).rows; + expect(rows.find(row => row.model.id === 'priced')?.status).toBe('metadata_fit'); + for (const id of ['missing', 'zero', 'invalid']) { + const row = rows.find(candidate => candidate.model.id === id); + expect(row?.status).toBe('unknown'); + expect(row?.estimatedCostUSD).toBeNull(); + expect(row?.reasons).toContain('unknown_price'); + } + }); + + it('excludes models over a specified budget while sorting remaining candidates by estimated cost', () => { + const report = assessModelFeasibility( + [ + model('expensive', { cost_per_1m_input: 100, cost_per_1m_output: 100 }), + model('cheap', { cost_per_1m_input: 0.5, cost_per_1m_output: 2 }), + model('middle', { cost_per_1m_input: 1, cost_per_1m_output: 3 }), + ], + req({ maxEstimatedCostUSD: 0.01 }) + ); + expect(report.rows.map(row => row.model.id)).toEqual(['cheap', 'middle', 'expensive']); + expect(report.counts).toEqual({ metadata_fit: 2, unknown: 0, excluded: 1 }); + expect(report.availabilityVerified).toBe(false); + expect(report.livePricesVerified).toBe(false); + expect(report.selectionIsAdvisory).toBe(true); + }); + + it('keeps missing context or vision capability unknown, without interpreting unknown as eligible', () => { + const report = assessModelFeasibility( + [ + model('missing-context', { context_window: undefined, vision: true }), + model('missing-vision', { vision: null }), + model('fit', { vision: true }), + ], + req({ requiresVision: true }) + ); + expect(report.rows.find(row => row.model.id === 'missing-context')?.status).toBe('unknown'); + expect(report.rows.find(row => row.model.id === 'missing-vision')?.status).toBe('unknown'); + expect(report.rows[0].model.id).toBe('fit'); + }); + + it('rejects invalid requirements and remains stable under reordered equal-price candidates', () => { + for (const fields of [ + { inputTokens: -1 }, + { outputTokens: Number.POSITIVE_INFINITY }, + { maxEstimatedCostUSD: -1 }, + { inputTokens: 0.1 }, + ]) { + expect(() => assessModelFeasibility([model('x')], req(fields))).toThrow(/Invalid/); + } + const first = [model('z'), model('b'), model('a')]; + const a = assessModelFeasibility(first, req()); + const b = assessModelFeasibility([...first].reverse(), req()); + expect(a.rows.map(row => row.model.id)).toEqual(['a', 'b', 'z']); + expect(a.rows.map(row => row.model.id)).toEqual(b.rows.map(row => row.model.id)); + }); +}); + +describe('conservative model cost presentation', () => { + it('does not display positive microcosts as zero or unknown', () => { + expect(formatEstimatedModelCost(0.0000004)).toBe('<$0.0001'); + expect(formatEstimatedModelCost(0)).toBe('Est. $0.0000'); + expect(formatEstimatedModelCost(null)).toBe('Cost unknown'); + expect(formatEstimatedModelCost(0.012345)).toBe('Est. $0.0123'); + }); +}); + +describe('one-sided and zero-token cost estimates', () => { + it('does not require input pricing for an output-only request', () => { + const result = assessModelFeasibility( + [model('output-only', { cost_per_1m_input: undefined, cost_per_1m_output: 5 })], + req({ inputTokens: 0, outputTokens: 1000 }) + ); + expect(result.rows[0].estimatedCostUSD).toBe(0.005); + expect(result.rows[0].status).toBe('metadata_fit'); + }); + + it('does not require output pricing for an input-only request', () => { + const result = assessModelFeasibility( + [model('input-only', { cost_per_1m_input: 2, cost_per_1m_output: null })], + req({ inputTokens: 1000, outputTokens: 0 }) + ); + expect(result.rows[0].estimatedCostUSD).toBe(0.002); + expect(result.rows[0].status).toBe('metadata_fit'); + }); + + it('estimates zero cost for zero usage even when price metadata is absent', () => { + const result = assessModelFeasibility( + [model('zero-usage', { cost_per_1m_input: undefined, cost_per_1m_output: null })], + req({ inputTokens: 0, outputTokens: 0, maxEstimatedCostUSD: 0 }) + ); + expect(result.rows[0].estimatedCostUSD).toBe(0); + expect(result.rows[0].status).toBe('metadata_fit'); + expect(result.rows[0].reasons).not.toContain('unknown_price'); + }); + + it('keeps pricing unknown when an actually used token component has no price', () => { + const result = assessModelFeasibility( + [model('unknown-input', { cost_per_1m_input: null, cost_per_1m_output: 5 })], + req({ inputTokens: 1000, outputTokens: 1000 }) + ); + expect(result.rows[0].estimatedCostUSD).toBeNull(); + expect(result.rows[0].status).toBe('unknown'); + expect(result.rows[0].reasons).toContain('unknown_price'); + }); +}); + +describe('cached-input token suitability (within total input)', () => { + it('separately prices uncached input, cached input, and output tokens', () => { + const result = assessModelFeasibility( + [ + model('cache', { + cost_per_1m_input: 10, + cost_per_1m_cached_input: 1, + cost_per_1m_output: 20, + }), + ], + req({ inputTokens: 1000, cachedInputTokens: 800, outputTokens: 100 }) + ); + expect(result.rows[0].estimatedCostUSD).toBeCloseTo(0.0048, 8); + expect(result.rows[0].status).toBe('metadata_fit'); + }); + + it('keeps the estimate unknown when cached tokens were used but their price is missing', () => { + const result = assessModelFeasibility( + [ + model('cache-price-unavailable', { + cost_per_1m_input: 10, + cost_per_1m_cached_input: undefined, + cost_per_1m_output: 20, + }), + ], + req({ inputTokens: 1000, cachedInputTokens: 800, outputTokens: 100 }) + ); + expect(result.rows[0].estimatedCostUSD).toBeNull(); + expect(result.rows[0].reasons).toContain('unknown_price'); + }); + + it('does not require cached pricing when cached-input usage is zero', () => { + const result = assessModelFeasibility( + [model('not-cached', { cost_per_1m_cached_input: undefined })], + req({ cachedInputTokens: 0 }) + ); + expect(result.rows[0].status).toBe('metadata_fit'); + expect(result.rows[0].estimatedCostUSD).toBe(0.005); + }); + + it('rejects impossible cached-input volumes and non-integer counts', () => { + for (const cachedInputTokens of [ + -1, + 1.5, + 1001, + Number.POSITIVE_INFINITY, + null as unknown as number, + ]) { + expect(() => + assessModelFeasibility([model('cache')], req({ inputTokens: 1000, cachedInputTokens })) + ).toThrow(/Invalid/); + } + }); +}); + +describe('saved context metadata validation', () => { + it('treats fractional, non-finite or unsafe context windows as unknown', () => { + for (const context_window of [ + 1.5, + Number.MAX_SAFE_INTEGER + 1, + Number.POSITIVE_INFINITY, + NaN, + ]) { + const result = assessModelFeasibility( + [model('invalid-context', { context_window })], + req({ inputTokens: 0, outputTokens: 0 }) + ); + expect(result.rows[0].status).toBe('unknown'); + expect(result.rows[0].reasons).toContain('unknown_context'); + } + }); +}); diff --git a/app/src/components/settings/panels/ai/modelFeasibility.ts b/app/src/components/settings/panels/ai/modelFeasibility.ts new file mode 100644 index 00000000000..abe7f46289b --- /dev/null +++ b/app/src/components/settings/panels/ai/modelFeasibility.ts @@ -0,0 +1,164 @@ +/** Advisory, no-egress model feasibility over SAVED catalog values only. + * This neither routes inference nor checks credentials or actual availability. + * Hard constraints precede cost ranking; missing or zero prices are unknown. + */ +export interface FeasibilityModel { + id: string; + provider: string; + context_window?: number | null; + vision?: boolean | null; + cost_per_1m_input?: number | null; + cost_per_1m_cached_input?: number | null; + cost_per_1m_output?: number | null; +} + +export interface ModelRequirements { + inputTokens: number; + cachedInputTokens?: number; + outputTokens: number; + requiresVision: boolean; + maxEstimatedCostUSD: number | null; +} + +export type ModelFitStatus = 'metadata_fit' | 'unknown' | 'excluded'; +export type ModelFitReason = + | 'context_too_small' + | 'vision_unavailable' + | 'estimated_cost_exceeds_budget' + | 'unknown_context' + | 'unknown_vision' + | 'unknown_price'; + +export interface ModelFitRow { + model: T; + status: ModelFitStatus; + reasons: ModelFitReason[]; + estimatedCostUSD: number | null; +} + +export interface ModelFeasibilityReport { + rows: ModelFitRow[]; + counts: Record; + availabilityVerified: false; + livePricesVerified: false; + selectionIsAdvisory: true; +} + +const knownNonnegative = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value) && value >= 0; + +/** Explainable metadata-only ranking; never claims provider eligibility. + * OpenHuman's ModelRegistryEntry contract uses zero/missing prices as unknown, + * including entries labeled "free" until authoritative pricing is checked. + */ +export function assessModelFeasibility( + models: readonly T[], + requirements: ModelRequirements +): ModelFeasibilityReport { + const { inputTokens, outputTokens, requiresVision, maxEstimatedCostUSD } = requirements; + const cachedInputTokens = + requirements.cachedInputTokens === undefined ? 0 : requirements.cachedInputTokens; + if ( + !Number.isSafeInteger(cachedInputTokens) || + cachedInputTokens < 0 || + cachedInputTokens > inputTokens || + !Number.isSafeInteger(inputTokens) || + inputTokens < 0 || + !Number.isSafeInteger(outputTokens) || + outputTokens < 0 || + typeof requiresVision !== 'boolean' || + (maxEstimatedCostUSD !== null && !knownNonnegative(maxEstimatedCostUSD)) + ) { + throw new Error('Invalid model feasibility requirements.'); + } + const demand = inputTokens + outputTokens; + if (!Number.isSafeInteger(demand)) { + throw new Error('Invalid combined token demand.'); + } + + const rows: ModelFitRow[] = models.map(model => { + const reasons: ModelFitReason[] = []; + const context = model.context_window; + if (!knownNonnegative(context) || !Number.isSafeInteger(context) || context === 0) { + reasons.push('unknown_context'); + } else if (context < demand) { + reasons.push('context_too_small'); + } + if (requiresVision) { + if (model.vision === false) { + reasons.push('vision_unavailable'); + } else if (model.vision !== true) { + reasons.push('unknown_vision'); + } + } + + const inputPrice = model.cost_per_1m_input; + const cachedInputPrice = model.cost_per_1m_cached_input; + const outputPrice = model.cost_per_1m_output; + const uncachedInputTokens = inputTokens - cachedInputTokens; + // Each consumed component needs a positive saved price; zero-price + // metadata with actual usage remains unknown, not verified free. + const uncachedKnown = + uncachedInputTokens === 0 || (knownNonnegative(inputPrice) && inputPrice > 0); + const cachedKnown = + cachedInputTokens === 0 || (knownNonnegative(cachedInputPrice) && cachedInputPrice > 0); + const outputKnown = outputTokens === 0 || (knownNonnegative(outputPrice) && outputPrice > 0); + const estimated = + uncachedKnown && cachedKnown && outputKnown + ? ((uncachedInputTokens === 0 ? 0 : uncachedInputTokens * (inputPrice as number)) + + (cachedInputTokens === 0 ? 0 : cachedInputTokens * (cachedInputPrice as number)) + + (outputTokens === 0 ? 0 : outputTokens * (outputPrice as number))) / + 1_000_000 + : null; + const estimatedCostUSD = estimated !== null && Number.isFinite(estimated) ? estimated : null; + if (estimatedCostUSD === null) { + reasons.push('unknown_price'); + } else if (maxEstimatedCostUSD !== null && estimatedCostUSD > maxEstimatedCostUSD) { + reasons.push('estimated_cost_exceeds_budget'); + } + + const excluded = reasons.some( + reason => + reason === 'context_too_small' || + reason === 'vision_unavailable' || + reason === 'estimated_cost_exceeds_budget' + ); + const status: ModelFitStatus = excluded + ? 'excluded' + : reasons.length > 0 + ? 'unknown' + : 'metadata_fit'; + return { model, status, reasons, estimatedCostUSD }; + }); + + const rank: Record = { metadata_fit: 0, unknown: 1, excluded: 2 }; + rows.sort((a, b) => { + const statusDiff = rank[a.status] - rank[b.status]; + if (statusDiff !== 0) return statusDiff; + if (a.estimatedCostUSD !== null && b.estimatedCostUSD !== null) { + const difference = a.estimatedCostUSD - b.estimatedCostUSD; + if (difference !== 0) return difference; + } else if (a.estimatedCostUSD !== null || b.estimatedCostUSD !== null) { + return a.estimatedCostUSD === null ? 1 : -1; + } + return a.model.provider.localeCompare(b.model.provider) || a.model.id.localeCompare(b.model.id); + }); + + const counts: Record = { metadata_fit: 0, unknown: 0, excluded: 0 }; + for (const row of rows) counts[row.status]++; + + return { + rows, + counts, + availabilityVerified: false, + livePricesVerified: false, + selectionIsAdvisory: true, + }; +} + +/** Present microcosts conservatively: positive usage must not look free. */ +export function formatEstimatedModelCost(cost: number | null): string { + if (cost === null || !Number.isFinite(cost) || cost < 0) return 'Cost unknown'; + if (cost > 0 && cost < 0.0001) return '<$0.0001'; + return 'Est. $' + cost.toFixed(4); +}