fix(eval): let coded agents pick the tool simulation model for BYOM tenants [AE-2294]

packages/uipath/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.14.25"
+version = "2.14.26"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

packages/uipath/src/uipath/eval/mocks/_input_mocker.py

@@ -10,11 +10,11 @@
 from uipath.core.tracing import traced
 from uipath.platform import UiPath
 from uipath.platform.chat import UiPathLlmChatService
-from uipath.platform.chat._llm_gateway_service import ChatModels
 
 from .._execution_context import eval_set_run_id_context
 from ._mock_context import cache_manager_context
 from ._mocker import UiPathInputMockingError, format_exception_message
+from ._simulation_model import simulation_completion_kwargs
 from ._structured_output import coerce_to_schema, generate_structured_output
 from ._types import (
     InputMockingStrategy,
@@ -127,17 +127,12 @@ async def generate_llm_input(
 
         prompt = get_input_mocking_prompt(**prompt_generation_args)
 
-        model_parameters = mocking_strategy.model if mocking_strategy else None
-        completion_kwargs = (
-            model_parameters.model_dump(by_alias=False, exclude_none=True)
-            if model_parameters
-            else {}
+        completion_kwargs = simulation_completion_kwargs(
+            mocking_strategy.model if mocking_strategy else None
         )
-
-        simulation_model = completion_kwargs.get(
-            "model", ChatModels.gpt_4_1_mini_2025_04_14
+        logger.info(
+            f"Simulating input generation using model: {completion_kwargs['model']}"
         )
-        logger.info(f"Simulating input generation using model: {simulation_model}")
 
         if cache_manager is not None:
             cache_key_data = {

packages/uipath/src/uipath/eval/mocks/_llm_mocker.py

@@ -11,7 +11,7 @@
 from uipath.core.tracing import traced
 from uipath.platform import UiPath
 from uipath.platform.chat import UiPathLlmChatService
-from uipath.platform.chat._llm_gateway_service import ChatModels, _cleanup_schema
+from uipath.platform.chat._llm_gateway_service import _cleanup_schema
 
 from .._execution_context import (
     eval_set_run_id_context,
@@ -30,6 +30,7 @@
     UiPathNoMockFoundError,
     format_exception_message,
 )
+from ._simulation_model import simulation_completion_kwargs
 from ._structured_output import generate_structured_output
 from ._types import (
     ExampleCall,
@@ -176,18 +177,12 @@ async def response(
                     k: json.dumps(pydantic_to_dict_safe(v), default=serialize_defaults)
                     for k, v in prompt_input.items()
                 }
-                model_parameters = self.context.strategy.model
-                completion_kwargs = (
-                    model_parameters.model_dump(by_alias=False, exclude_none=True)
-                    if model_parameters
-                    else {}
-                )
-
-                simulation_model = completion_kwargs.get(
-                    "model", ChatModels.gpt_4_1_mini_2025_04_14
+                completion_kwargs = simulation_completion_kwargs(
+                    self.context.strategy.model
                 )
                 logger.info(
-                    f"Simulating tool '{function_name}' using model: {simulation_model}"
+                    f"Simulating tool '{function_name}' using model: "
+                    f"{completion_kwargs['model']}"
                 )
 
                 formatted_prompt = PROMPT.format(**prompt_generation_args)

packages/uipath/src/uipath/eval/mocks/_simulation_model.py

@@ -0,0 +1,27 @@
+"""Model resolution for LLM-backed simulations."""
+
+import os
+from typing import Any
+
+from uipath.platform.chat._llm_gateway_service import ChatModels
+
+from ._types import ModelSettings
+
+SIMULATION_MODEL_ENV = "UIPATH_SIMULATION_MODEL"
+
+
+def simulation_completion_kwargs(
+    model_settings: ModelSettings | None,
+) -> dict[str, Any]:
+    """Build completion kwargs for a simulation call, always naming the model."""
+    completion_kwargs = (
+        model_settings.model_dump(by_alias=False, exclude_none=True)
+        if model_settings
+        else {}
+    )
+    # Coded agents have no model in their schema; BYOM tenants block the default.
+    completion_kwargs.setdefault(
+        "model",
+        os.environ.get(SIMULATION_MODEL_ENV) or ChatModels.gpt_4_1_mini_2025_04_14,
+    )
+    return completion_kwargs

packages/uipath/tests/cli/eval/mocks/test_simulation_model.py

@@ -0,0 +1,104 @@
+from unittest.mock import MagicMock
+
+import pytest
+from _pytest.monkeypatch import MonkeyPatch
+from pytest_httpx import HTTPXMock
+
+from uipath.eval.mocks import mockable
+from uipath.eval.mocks._cache_manager import CacheManager
+from uipath.eval.mocks._mock_runtime import (
+    clear_execution_context,
+    set_execution_context,
+)
+from uipath.eval.mocks._simulation_model import (
+    SIMULATION_MODEL_ENV,
+    simulation_completion_kwargs,
+)
+from uipath.eval.mocks._types import (
+    LLMMockingStrategy,
+    MockingContext,
+    ModelSettings,
+    ToolSimulation,
+)
+
+BYOM_MODEL = "Siemens-SDC-gpt-4o"
+
+
+def test_defaults_to_gateway_model_without_settings_or_env(monkeypatch: MonkeyPatch):
+    monkeypatch.delenv(SIMULATION_MODEL_ENV, raising=False)
+
+    assert simulation_completion_kwargs(None) == {"model": "gpt-4.1-mini-2025-04-14"}
+
+
+def test_env_model_applies_when_settings_have_no_model(monkeypatch: MonkeyPatch):
+    monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL)
+
+    assert simulation_completion_kwargs(None) == {"model": BYOM_MODEL}
+
+
+def test_configured_model_wins_over_env(monkeypatch: MonkeyPatch):
+    monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL)
+    settings = ModelSettings(model="gpt-4o-2024-11-20", temperature=0.2)
+
+    assert simulation_completion_kwargs(settings) == {
+        "model": "gpt-4o-2024-11-20",
+        "temperature": 0.2,
+    }
+
+
+@pytest.mark.httpx_mock(assert_all_responses_were_requested=False)
+def test_tool_simulation_sends_env_model(
+    httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch
+):
+    monkeypatch.setenv("UIPATH_URL", "https://example.com")
+    monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "1234567890")
+    monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL)
+    monkeypatch.setattr(CacheManager, "get", lambda *args, **kwargs: None)
+    monkeypatch.setattr(CacheManager, "set", lambda *args, **kwargs: None)
+
+    @mockable()
+    def web_search(*args, **kwargs) -> str:
+        raise NotImplementedError()
+
+    for service in ("agenthub_", "orchestrator_"):
+        httpx_mock.add_response(
+            url=f"https://example.com/{service}/llm/api/capabilities", json={}
+        )
+    httpx_mock.add_response(
+        url="https://example.com/llm/api/chat/completions"
+        "?api-version=2024-08-01-preview",
+        json={
+            "id": "response-id",
+            "object": "",
+            "created": 0,
+            "model": BYOM_MODEL,
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {"role": "ai", "content": '"ok"', "tool_calls": None},
+                    "finish_reason": "EOS",
+                }
+            ],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+        },
+    )
+    # Mirrors what cli_run/cli_debug build for a coded agent: no model anywhere.
+    set_execution_context(
+        MockingContext(
+            strategy=LLMMockingStrategy(
+                prompt="",
+                tools_to_simulate=[ToolSimulation(name="web_search")],
+            ),
+            name="debug-simulation",
+        ),
+        MagicMock(),
+        "test-execution-id",
+    )
+    try:
+        assert web_search() == "ok"
+    finally:
+        clear_execution_context()
+
+    request = httpx_mock.get_request(method="POST")
+    assert request is not None
+    assert request.headers["X-UiPath-LlmGateway-NormalizedApi-ModelName"] == BYOM_MODEL