Skip to content
This repository was archived by the owner on Oct 16, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ jobs:
run: bun run contract-eval
env:
GH_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }}
CONTRACT_EVAL_MODEL: ${{ vars.CONTRACT_EVAL_MODEL || 'openai/gpt-4.1-mini' }}
GH_MODELS_MODEL: ${{ vars.GH_MODELS_MODEL || 'gpt-5-mini' }}

- name: Bump version, commit, and tag
if: inputs.channel != 'existing'
Expand Down Expand Up @@ -229,7 +229,7 @@ jobs:
run: bun run contract-eval
env:
GH_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }}
CONTRACT_EVAL_MODEL: ${{ vars.CONTRACT_EVAL_MODEL || 'openai/gpt-4.1-mini' }}
GH_MODELS_MODEL: ${{ vars.GH_MODELS_MODEL || 'gpt-5-mini' }}

- name: Publish to npm
run: |
Expand Down
2 changes: 1 addition & 1 deletion examples/contract/evals/release-gate.eval.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ environment:
type: host
workdir: ../workspace-template
providers:
- github-models-contract
- gh-models
prompts:
- "{{ input }}"
tests:
Expand Down
2 changes: 1 addition & 1 deletion examples/contract/evals/repo-materialization.eval.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ environment:
- 21a34daed7ebcfe36cbed053607622a55e5e94cb
cwd: "."
providers:
- github-models-contract
- gh-models
prompts:
- "{{ input }}"
tests:
Expand Down
2 changes: 1 addition & 1 deletion examples/contract/evals/script-grader-contract.eval.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: script-grader-contract
description: Release gate verifying the script-grader stdin payload contract.
providers:
- github-models-contract
- gh-models
prompts:
- "{{ input }}"
tests:
Expand Down
8 changes: 5 additions & 3 deletions scripts/run-contract-eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@
* GH_MODELS_TOKEN=$(gh auth token) bun run contract-eval
*/

process.env.CONTRACT_EVAL_MODEL ||= 'openai/gpt-4.1-mini';
process.env.GH_MODELS_MODEL ||= 'gpt-5-mini';

const evalFiles = [
'examples/contract/evals/release-gate.eval.yaml',
'examples/contract/evals/repo-materialization.eval.yaml',
'examples/contract/evals/code-grader-contract.eval.yaml',
'examples/contract/evals/script-grader-contract.eval.yaml',
];

for (const evalFile of evalFiles) {
Expand All @@ -27,7 +27,9 @@ for (const evalFile of evalFiles) {
'eval',
evalFile,
'--provider',
'github-models-contract',
'gh-models',
'--grader-provider',
'gh-models',
'--threshold',
'1',
],
Expand Down
Loading