diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 846caa1c5..47c82448e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -107,7 +107,7 @@ jobs: run: bun run contract-eval env: GH_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }} - CONTRACT_EVAL_MODEL: ${{ vars.CONTRACT_EVAL_MODEL || 'openai/gpt-4.1-mini' }} + GH_MODELS_MODEL: ${{ vars.GH_MODELS_MODEL || 'gpt-5-mini' }} - name: Bump version, commit, and tag if: inputs.channel != 'existing' @@ -229,7 +229,7 @@ jobs: run: bun run contract-eval env: GH_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }} - CONTRACT_EVAL_MODEL: ${{ vars.CONTRACT_EVAL_MODEL || 'openai/gpt-4.1-mini' }} + GH_MODELS_MODEL: ${{ vars.GH_MODELS_MODEL || 'gpt-5-mini' }} - name: Publish to npm run: | diff --git a/examples/contract/evals/release-gate.eval.yaml b/examples/contract/evals/release-gate.eval.yaml index 103aebc5e..fdc0fe9da 100644 --- a/examples/contract/evals/release-gate.eval.yaml +++ b/examples/contract/evals/release-gate.eval.yaml @@ -4,7 +4,7 @@ environment: type: host workdir: ../workspace-template providers: - - github-models-contract + - gh-models prompts: - "{{ input }}" tests: diff --git a/examples/contract/evals/repo-materialization.eval.yaml b/examples/contract/evals/repo-materialization.eval.yaml index 9aa7b1904..eefe618dd 100644 --- a/examples/contract/evals/repo-materialization.eval.yaml +++ b/examples/contract/evals/repo-materialization.eval.yaml @@ -13,7 +13,7 @@ environment: - 21a34daed7ebcfe36cbed053607622a55e5e94cb cwd: "." providers: - - github-models-contract + - gh-models prompts: - "{{ input }}" tests: diff --git a/examples/contract/evals/script-grader-contract.eval.yaml b/examples/contract/evals/script-grader-contract.eval.yaml index 24a5fe0cb..bf0312691 100644 --- a/examples/contract/evals/script-grader-contract.eval.yaml +++ b/examples/contract/evals/script-grader-contract.eval.yaml @@ -1,7 +1,7 @@ name: script-grader-contract description: Release gate verifying the script-grader stdin payload contract. providers: - - github-models-contract + - gh-models prompts: - "{{ input }}" tests: diff --git a/scripts/run-contract-eval.ts b/scripts/run-contract-eval.ts index d6ff811f8..fe3b0f2f8 100644 --- a/scripts/run-contract-eval.ts +++ b/scripts/run-contract-eval.ts @@ -10,12 +10,12 @@ * GH_MODELS_TOKEN=$(gh auth token) bun run contract-eval */ -process.env.CONTRACT_EVAL_MODEL ||= 'openai/gpt-4.1-mini'; +process.env.GH_MODELS_MODEL ||= 'gpt-5-mini'; const evalFiles = [ 'examples/contract/evals/release-gate.eval.yaml', 'examples/contract/evals/repo-materialization.eval.yaml', - 'examples/contract/evals/code-grader-contract.eval.yaml', + 'examples/contract/evals/script-grader-contract.eval.yaml', ]; for (const evalFile of evalFiles) { @@ -27,7 +27,9 @@ for (const evalFile of evalFiles) { 'eval', evalFile, '--provider', - 'github-models-contract', + 'gh-models', + '--grader-provider', + 'gh-models', '--threshold', '1', ],