From 2eab25421657594eba78b0bebd63fec22c5ffcfd Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Sat, 10 Oct 2026 18:30:21 +1100
Subject: [PATCH] fix(release): use Azure for contract evals

---
 .github/workflows/publish.yml                      | 14 ++++++++------
 examples/contract/evals/release-gate.eval.yaml     |  2 +-
 .../contract/evals/repo-materialization.eval.yaml  |  2 +-
 .../evals/script-grader-contract.eval.yaml         |  2 +-
 scripts/run-contract-eval.ts                       |  9 ++++-----
 5 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 47c82448e..b7d73e94b 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -100,14 +100,15 @@ jobs:
         run: bun install --frozen-lockfile
 
       # Finalize/stable releases promote to latest, so block before any version
-      # bump/tag if the live contract eval fails. The gate uses a Models-capable
-      # PAT secret and does not override the workflow GITHUB_TOKEN.
+      # bump/tag if the live contract eval fails. The gate uses the configured
+      # Azure OpenAI deployment and does not override the workflow GITHUB_TOKEN.
       - name: Contract eval gate
         if: inputs.channel == 'finalize' || inputs.channel == 'stable'
         run: bun run contract-eval
         env:
-          GH_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }}
-          GH_MODELS_MODEL: ${{ vars.GH_MODELS_MODEL || 'gpt-5-mini' }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
+          AZURE_DEPLOYMENT_NAME: ${{ vars.AZURE_DEPLOYMENT_NAME }}
 
       - name: Bump version, commit, and tag
         if: inputs.channel != 'existing'
@@ -228,8 +229,9 @@ jobs:
         if: needs.prepare.outputs.publish_action == 'publish-latest' && inputs.channel == 'existing'
         run: bun run contract-eval
         env:
-          GH_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }}
-          GH_MODELS_MODEL: ${{ vars.GH_MODELS_MODEL || 'gpt-5-mini' }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
+          AZURE_DEPLOYMENT_NAME: ${{ vars.AZURE_DEPLOYMENT_NAME }}
 
       - name: Publish to npm
         run: |
diff --git a/examples/contract/evals/release-gate.eval.yaml b/examples/contract/evals/release-gate.eval.yaml
index fdc0fe9da..ed5dcd6e6 100644
--- a/examples/contract/evals/release-gate.eval.yaml
+++ b/examples/contract/evals/release-gate.eval.yaml
@@ -4,7 +4,7 @@ environment:
   type: host
   workdir: ../workspace-template
 providers:
-  - gh-models
+  - azure
 prompts:
   - "{{ input }}"
 tests:
diff --git a/examples/contract/evals/repo-materialization.eval.yaml b/examples/contract/evals/repo-materialization.eval.yaml
index eefe618dd..01f5c2873 100644
--- a/examples/contract/evals/repo-materialization.eval.yaml
+++ b/examples/contract/evals/repo-materialization.eval.yaml
@@ -13,7 +13,7 @@ environment:
       - 21a34daed7ebcfe36cbed053607622a55e5e94cb
     cwd: "."
 providers:
-  - gh-models
+  - azure
 prompts:
   - "{{ input }}"
 tests:
diff --git a/examples/contract/evals/script-grader-contract.eval.yaml b/examples/contract/evals/script-grader-contract.eval.yaml
index bf0312691..26a5b9b6b 100644
--- a/examples/contract/evals/script-grader-contract.eval.yaml
+++ b/examples/contract/evals/script-grader-contract.eval.yaml
@@ -1,7 +1,7 @@
 name: script-grader-contract
 description: Release gate verifying the script-grader stdin payload contract.
 providers:
-  - gh-models
+  - azure
 prompts:
   - "{{ input }}"
 tests:
diff --git a/scripts/run-contract-eval.ts b/scripts/run-contract-eval.ts
index fe3b0f2f8..7416cbd3f 100644
--- a/scripts/run-contract-eval.ts
+++ b/scripts/run-contract-eval.ts
@@ -7,11 +7,10 @@
  * invoking this file directly.
  *
  * Local usage:
- * GH_MODELS_TOKEN=$(gh auth token) bun run contract-eval
+ * Set AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, and AZURE_DEPLOYMENT_NAME,
+ * then run `bun run contract-eval`.
  */
 
-process.env.GH_MODELS_MODEL ||= 'gpt-5-mini';
-
 const evalFiles = [
   'examples/contract/evals/release-gate.eval.yaml',
   'examples/contract/evals/repo-materialization.eval.yaml',
@@ -27,9 +26,9 @@ for (const evalFile of evalFiles) {
       'eval',
       evalFile,
       '--provider',
-      'gh-models',
+      'azure',
       '--grader-provider',
-      'gh-models',
+      'azure',
       '--threshold',
       '1',
     ],