Skip to content
This repository was archived by the owner on Oct 16, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,13 @@ jobs:

# Finalize/stable releases promote to latest, so block before any version
# bump/tag if the live contract eval fails. The gate uses the configured
# Azure OpenAI deployment and does not override the workflow GITHUB_TOKEN.
# OpenRouter model and does not override the workflow GITHUB_TOKEN.
- name: Contract eval gate
if: inputs.channel == 'finalize' || inputs.channel == 'stable'
run: bun run contract-eval
env:
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
AZURE_DEPLOYMENT_NAME: ${{ vars.AZURE_DEPLOYMENT_NAME }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
OPENROUTER_MODEL: ${{ vars.OPENROUTER_MODEL || 'openai/gpt-5.4-mini' }}

- name: Bump version, commit, and tag
if: inputs.channel != 'existing'
Expand Down Expand Up @@ -229,9 +228,8 @@ jobs:
if: needs.prepare.outputs.publish_action == 'publish-latest' && inputs.channel == 'existing'
run: bun run contract-eval
env:
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
AZURE_DEPLOYMENT_NAME: ${{ vars.AZURE_DEPLOYMENT_NAME }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
OPENROUTER_MODEL: ${{ vars.OPENROUTER_MODEL || 'openai/gpt-5.4-mini' }}

- name: Publish to npm
run: |
Expand Down
2 changes: 1 addition & 1 deletion examples/contract/evals/release-gate.eval.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ environment:
type: host
workdir: ../workspace-template
providers:
- azure
- openrouter
prompts:
- "{{ input }}"
tests:
Expand Down
2 changes: 1 addition & 1 deletion examples/contract/evals/repo-materialization.eval.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ environment:
- 21a34daed7ebcfe36cbed053607622a55e5e94cb
cwd: "."
providers:
- azure
- openrouter
prompts:
- "{{ input }}"
tests:
Expand Down
2 changes: 1 addition & 1 deletion examples/contract/evals/script-grader-contract.eval.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: script-grader-contract
description: Release gate verifying the script-grader stdin payload contract.
providers:
- azure
- openrouter
prompts:
- "{{ input }}"
tests:
Expand Down
6 changes: 3 additions & 3 deletions scripts/run-contract-eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* invoking this file directly.
*
* Local usage:
* Set AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, and AZURE_DEPLOYMENT_NAME,
* Set OPENROUTER_API_KEY and OPENROUTER_MODEL,
* then run `bun run contract-eval`.
*/

Expand All @@ -26,9 +26,9 @@ for (const evalFile of evalFiles) {
'eval',
evalFile,
'--provider',
'azure',
'openrouter',
'--grader-provider',
'azure',
'openrouter',
'--threshold',
'1',
],
Expand Down
Loading