From d1fdd3925bf8cd32bb5675e2d9fb83e39d794967 Mon Sep 17 00:00:00 2001 From: Chibi Date: Mon, 28 Sep 2026 09:33:50 -0700 Subject: [PATCH] fix(eval): let coded agents pick the tool simulation model for BYOM tenants [AE-2294] Tool and input simulation fell back to a hardcoded gpt-4.1-mini whenever the strategy had no model. Coded agents never carry a model in their runtime schema, so every coded agent simulation hit the UiPath-hosted default, which BYOM-only governance policies block. UIPATH_SIMULATION_MODEL now fills that gap, and the resolved model is actually sent instead of only being logged. Co-Authored-By: Claude Opus 5.5 --- packages/uipath/pyproject.toml | 2 +- .../src/uipath/eval/mocks/_input_mocker.py | 15 +-- .../src/uipath/eval/mocks/_llm_mocker.py | 17 +-- .../uipath/eval/mocks/_simulation_model.py | 27 +++++ .../cli/eval/mocks/test_simulation_model.py | 104 ++++++++++++++++++ packages/uipath/uv.lock | 6 +- 6 files changed, 146 insertions(+), 25 deletions(-) create mode 100644 packages/uipath/src/uipath/eval/mocks/_simulation_model.py create mode 100644 packages/uipath/tests/cli/eval/mocks/test_simulation_model.py diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index bb12bfa6f..1ce573930 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.14.25" +version = "2.14.26" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/mocks/_input_mocker.py b/packages/uipath/src/uipath/eval/mocks/_input_mocker.py index 381a4f071..275d69bdb 100644 --- a/packages/uipath/src/uipath/eval/mocks/_input_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_input_mocker.py @@ -10,11 +10,11 @@ from uipath.core.tracing import traced from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService -from uipath.platform.chat._llm_gateway_service import ChatModels from .._execution_context import eval_set_run_id_context from ._mock_context import cache_manager_context from ._mocker import UiPathInputMockingError, format_exception_message +from ._simulation_model import simulation_completion_kwargs from ._structured_output import coerce_to_schema, generate_structured_output from ._types import ( InputMockingStrategy, @@ -127,17 +127,12 @@ async def generate_llm_input( prompt = get_input_mocking_prompt(**prompt_generation_args) - model_parameters = mocking_strategy.model if mocking_strategy else None - completion_kwargs = ( - model_parameters.model_dump(by_alias=False, exclude_none=True) - if model_parameters - else {} + completion_kwargs = simulation_completion_kwargs( + mocking_strategy.model if mocking_strategy else None ) - - simulation_model = completion_kwargs.get( - "model", ChatModels.gpt_4_1_mini_2025_04_14 + logger.info( + f"Simulating input generation using model: {completion_kwargs['model']}" ) - logger.info(f"Simulating input generation using model: {simulation_model}") if cache_manager is not None: cache_key_data = { diff --git a/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py b/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py index 559a2e32f..853f64dba 100644 --- a/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py @@ -11,7 +11,7 @@ from uipath.core.tracing import traced from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService -from uipath.platform.chat._llm_gateway_service import ChatModels, _cleanup_schema +from uipath.platform.chat._llm_gateway_service import _cleanup_schema from .._execution_context import ( eval_set_run_id_context, @@ -30,6 +30,7 @@ UiPathNoMockFoundError, format_exception_message, ) +from ._simulation_model import simulation_completion_kwargs from ._structured_output import generate_structured_output from ._types import ( ExampleCall, @@ -176,18 +177,12 @@ async def response( k: json.dumps(pydantic_to_dict_safe(v), default=serialize_defaults) for k, v in prompt_input.items() } - model_parameters = self.context.strategy.model - completion_kwargs = ( - model_parameters.model_dump(by_alias=False, exclude_none=True) - if model_parameters - else {} - ) - - simulation_model = completion_kwargs.get( - "model", ChatModels.gpt_4_1_mini_2025_04_14 + completion_kwargs = simulation_completion_kwargs( + self.context.strategy.model ) logger.info( - f"Simulating tool '{function_name}' using model: {simulation_model}" + f"Simulating tool '{function_name}' using model: " + f"{completion_kwargs['model']}" ) formatted_prompt = PROMPT.format(**prompt_generation_args) diff --git a/packages/uipath/src/uipath/eval/mocks/_simulation_model.py b/packages/uipath/src/uipath/eval/mocks/_simulation_model.py new file mode 100644 index 000000000..57195c210 --- /dev/null +++ b/packages/uipath/src/uipath/eval/mocks/_simulation_model.py @@ -0,0 +1,27 @@ +"""Model resolution for LLM-backed simulations.""" + +import os +from typing import Any + +from uipath.platform.chat._llm_gateway_service import ChatModels + +from ._types import ModelSettings + +SIMULATION_MODEL_ENV = "UIPATH_SIMULATION_MODEL" + + +def simulation_completion_kwargs( + model_settings: ModelSettings | None, +) -> dict[str, Any]: + """Build completion kwargs for a simulation call, always naming the model.""" + completion_kwargs = ( + model_settings.model_dump(by_alias=False, exclude_none=True) + if model_settings + else {} + ) + # Coded agents have no model in their schema; BYOM tenants block the default. + completion_kwargs.setdefault( + "model", + os.environ.get(SIMULATION_MODEL_ENV) or ChatModels.gpt_4_1_mini_2025_04_14, + ) + return completion_kwargs diff --git a/packages/uipath/tests/cli/eval/mocks/test_simulation_model.py b/packages/uipath/tests/cli/eval/mocks/test_simulation_model.py new file mode 100644 index 000000000..f02154493 --- /dev/null +++ b/packages/uipath/tests/cli/eval/mocks/test_simulation_model.py @@ -0,0 +1,104 @@ +from unittest.mock import MagicMock + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from pytest_httpx import HTTPXMock + +from uipath.eval.mocks import mockable +from uipath.eval.mocks._cache_manager import CacheManager +from uipath.eval.mocks._mock_runtime import ( + clear_execution_context, + set_execution_context, +) +from uipath.eval.mocks._simulation_model import ( + SIMULATION_MODEL_ENV, + simulation_completion_kwargs, +) +from uipath.eval.mocks._types import ( + LLMMockingStrategy, + MockingContext, + ModelSettings, + ToolSimulation, +) + +BYOM_MODEL = "Siemens-SDC-gpt-4o" + + +def test_defaults_to_gateway_model_without_settings_or_env(monkeypatch: MonkeyPatch): + monkeypatch.delenv(SIMULATION_MODEL_ENV, raising=False) + + assert simulation_completion_kwargs(None) == {"model": "gpt-4.1-mini-2025-04-14"} + + +def test_env_model_applies_when_settings_have_no_model(monkeypatch: MonkeyPatch): + monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL) + + assert simulation_completion_kwargs(None) == {"model": BYOM_MODEL} + + +def test_configured_model_wins_over_env(monkeypatch: MonkeyPatch): + monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL) + settings = ModelSettings(model="gpt-4o-2024-11-20", temperature=0.2) + + assert simulation_completion_kwargs(settings) == { + "model": "gpt-4o-2024-11-20", + "temperature": 0.2, + } + + +@pytest.mark.httpx_mock(assert_all_responses_were_requested=False) +def test_tool_simulation_sends_env_model( + httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch +): + monkeypatch.setenv("UIPATH_URL", "https://example.com") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "1234567890") + monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL) + monkeypatch.setattr(CacheManager, "get", lambda *args, **kwargs: None) + monkeypatch.setattr(CacheManager, "set", lambda *args, **kwargs: None) + + @mockable() + def web_search(*args, **kwargs) -> str: + raise NotImplementedError() + + for service in ("agenthub_", "orchestrator_"): + httpx_mock.add_response( + url=f"https://example.com/{service}/llm/api/capabilities", json={} + ) + httpx_mock.add_response( + url="https://example.com/llm/api/chat/completions" + "?api-version=2024-08-01-preview", + json={ + "id": "response-id", + "object": "", + "created": 0, + "model": BYOM_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "ai", "content": '"ok"', "tool_calls": None}, + "finish_reason": "EOS", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + # Mirrors what cli_run/cli_debug build for a coded agent: no model anywhere. + set_execution_context( + MockingContext( + strategy=LLMMockingStrategy( + prompt="", + tools_to_simulate=[ToolSimulation(name="web_search")], + ), + name="debug-simulation", + ), + MagicMock(), + "test-execution-id", + ) + try: + assert web_search() == "ok" + finally: + clear_execution_context() + + request = httpx_mock.get_request(method="POST") + assert request is not None + assert request.headers["X-UiPath-LlmGateway-NormalizedApi-ModelName"] == BYOM_MODEL diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index bc92233bb..04b2fe80d 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -7,10 +7,10 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P2D" [options.exclude-newer-package] +uipath-core = false uipath-ipc = false -uipath-runtime = false uipath-platform = false -uipath-core = false +uipath-runtime = false [[package]] name = "aiohappyeyeballs" @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.14.25" +version = "2.14.26" source = { editable = "." } dependencies = [ { name = "applicationinsights" },