From fb8b1354775395db23509654fa2ad528740e70a2 Mon Sep 17 00:00:00 2001 From: suhasagg Date: Wed, 28 Oct 2026 14:03:09 +0530 Subject: [PATCH 1/2] fix(python): deduplicate CodeAct FIDES compatibility warnings --- .../_execute_code_tool.py | 48 +++++++++++++---- .../hyperlight/test_hyperlight_codeact.py | 54 +++++++++++++++++++ .../_execute_code_tool.py | 48 +++++++++++++---- .../monty/tests/monty/test_monty_codeact.py | 54 +++++++++++++++++++ 4 files changed, 184 insertions(+), 20 deletions(-) diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index c02ce4fdeb7..802144761af 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -1531,6 +1531,40 @@ def _build_sandbox() -> tuple[Any, Any]: ) +def _fides_metadata_snapshot(tool_obj: FunctionTool) -> dict[str, str]: + """Capture FIDES metadata for compatibility-warning deduplication.""" + properties = tool_obj.additional_properties or {} + return {key: repr(value) for key, value in properties.items() if key in _FIDES_TOOL_PROPERTY_KEYS} + + +def _warn_unsupported_fides_tools( + tools: Sequence[FunctionTool], + warned_metadata: list[tuple[FunctionTool, dict[str, str]]], +) -> None: + """Warn when newly registered or changed FIDES metadata is observed.""" + for tool_obj in tools: + snapshot = _fides_metadata_snapshot(tool_obj) + previous = next( + (metadata for registered, metadata in warned_metadata if registered is tool_obj), + None, + ) + + if snapshot and snapshot != previous: + logger.warning( + "FIDES is not supported with CodeAct providers. Metadata on tool '%s' will not be enforced " + "through function middleware for calls made inside generated code. " + "Keep FIDES-dependent tools as direct agent tools.", + tool_obj.name, + ) + + warned_metadata[:] = [ + (registered, metadata) for registered, metadata in warned_metadata if registered is not tool_obj + ] + + if snapshot: + warned_metadata.append((tool_obj, snapshot)) + + class HyperlightExecuteCodeTool(FunctionTool): """Execute Python code inside a Hyperlight sandbox. @@ -1557,6 +1591,7 @@ def __init__( module: str | None = DEFAULT_HYPERLIGHT_MODULE, module_path: str | None = None, _registry: SandboxRuntime | None = None, + _fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] | None = None, ) -> None: normalized_description_format = _normalize_tool_description_format(tool_description_format) max_output_files = _validate_positive_integer(name="max_output_files", value=max_output_files) @@ -1581,6 +1616,7 @@ def __init__( self._max_output_file_bytes = max_output_file_bytes self._max_output_total_bytes = max_output_total_bytes self._managed_tools: list[FunctionTool] = [] + self._fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] = list(_fides_warned_metadata or []) self._file_mounts: dict[str, FileMount] = {} self._allowed_domains: dict[str, AllowedDomain] = {} @@ -1623,16 +1659,7 @@ def add_tools( new_tools = _collect_tools(tools) combined_tools = _collect_tools(self._managed_tools, new_tools) self._managed_tools = combined_tools - for tool_obj in new_tools: - if tool_obj.additional_properties and not _FIDES_TOOL_PROPERTY_KEYS.isdisjoint( - tool_obj.additional_properties - ): - logger.warning( - "FIDES is not supported with CodeAct providers. Metadata on tool '%s' will not be enforced " - "through function middleware for calls made inside generated code. " - "Keep FIDES-dependent tools as direct agent tools.", - tool_obj.name, - ) + _warn_unsupported_fides_tools(new_tools, self._fides_warned_metadata) self._refresh_approval_mode() def get_tools(self) -> list[FunctionTool]: @@ -1739,6 +1766,7 @@ def create_run_tool(self) -> HyperlightExecuteCodeTool: allowed_domains = self.get_allowed_domains() return HyperlightExecuteCodeTool( + _fides_warned_metadata=self._fides_warned_metadata, tools=self.get_tools(), tool_description_format=self._tool_description_format, approval_mode=self._default_approval_mode, diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index e15af170fba..dd9c6c61ebd 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -3501,3 +3501,57 @@ async def _drive_many(tool: HyperlightExecuteCodeTool) -> None: gc.collect() assert drop_violations == [] + + +def test_run_snapshot_does_not_repeat_unchanged_fides_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = HyperlightExecuteCodeTool(tools=annotated_tool, _registry=_FakeRuntime()) + + caplog.clear() + + with caplog.at_level("WARNING", logger="agent_framework"): + codeact.create_run_tool() + codeact.create_run_tool() + + assert "FIDES is not supported with CodeAct providers." not in caplog.text + + +def test_run_snapshot_warns_when_fides_metadata_keys_change( + caplog: pytest.LogCaptureFixture, +) -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = HyperlightExecuteCodeTool(tools=annotated_tool, _registry=_FakeRuntime()) + assert annotated_tool.additional_properties is not None + annotated_tool.additional_properties["confidentiality"] = "private" + + with caplog.at_level("WARNING", logger="agent_framework"): + codeact.create_run_tool() + + assert "FIDES is not supported with CodeAct providers." in caplog.text + assert codeact.get_tools()[0] is annotated_tool + + +def test_run_snapshot_warns_when_fides_metadata_value_changes(caplog: pytest.LogCaptureFixture) -> None: + @tool(additional_properties={"source_integrity": "trusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = HyperlightExecuteCodeTool(tools=annotated_tool, _registry=_FakeRuntime()) + assert annotated_tool.additional_properties is not None + + caplog.clear() + annotated_tool.additional_properties["source_integrity"] = "untrusted" + + with caplog.at_level("WARNING", logger="agent_framework"): + run_tool = codeact.create_run_tool() + + assert "FIDES is not supported with CodeAct providers." in caplog.text + assert run_tool.get_tools()[0] is annotated_tool diff --git a/python/packages/monty/agent_framework_monty/_execute_code_tool.py b/python/packages/monty/agent_framework_monty/_execute_code_tool.py index 551c5964441..46b42798419 100644 --- a/python/packages/monty/agent_framework_monty/_execute_code_tool.py +++ b/python/packages/monty/agent_framework_monty/_execute_code_tool.py @@ -186,6 +186,40 @@ def _make_tool_callback(tool_obj: FunctionTool) -> Callable[..., Any]: return partial(tool_obj.invoke, skip_parsing=True) +def _fides_metadata_snapshot(tool_obj: FunctionTool) -> dict[str, str]: + """Capture FIDES metadata for compatibility-warning deduplication.""" + properties = tool_obj.additional_properties or {} + return {key: repr(value) for key, value in properties.items() if key in _FIDES_TOOL_PROPERTY_KEYS} + + +def _warn_unsupported_fides_tools( + tools: Sequence[FunctionTool], + warned_metadata: list[tuple[FunctionTool, dict[str, str]]], +) -> None: + """Warn when newly registered or changed FIDES metadata is observed.""" + for tool_obj in tools: + snapshot = _fides_metadata_snapshot(tool_obj) + previous = next( + (metadata for registered, metadata in warned_metadata if registered is tool_obj), + None, + ) + + if snapshot and snapshot != previous: + logger.warning( + "FIDES is not supported with CodeAct providers. Metadata on tool '%s' will not be enforced " + "through function middleware for calls made inside generated code. " + "Keep FIDES-dependent tools as direct agent tools.", + tool_obj.name, + ) + + warned_metadata[:] = [ + (registered, metadata) for registered, metadata in warned_metadata if registered is not tool_obj + ] + + if snapshot: + warned_metadata.append((tool_obj, snapshot)) + + class MontyExecuteCodeTool(FunctionTool): """Execute Python code inside a Monty interpreter. @@ -229,6 +263,7 @@ def __init__( file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, resource_limits: dict[str, Any] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + _fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] | None = None, ) -> None: super().__init__( name=EXECUTE_CODE_TOOL_NAME, @@ -242,6 +277,7 @@ def __init__( tool_description_format ) self._managed_tools: list[FunctionTool] = [] + self._fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] = list(_fides_warned_metadata or []) self._workspace_root: Path | None = ( _resolve_existing_directory(workspace_root) if workspace_root is not None else None ) @@ -277,16 +313,7 @@ def add_tools( """Add Monty-side tools to this execute_code surface.""" new_tools = _collect_tools(tools) self._managed_tools = _collect_tools(self._managed_tools, new_tools) - for tool_obj in new_tools: - if tool_obj.additional_properties and not _FIDES_TOOL_PROPERTY_KEYS.isdisjoint( - tool_obj.additional_properties - ): - logger.warning( - "FIDES is not supported with CodeAct providers. Metadata on tool '%s' will not be enforced " - "through function middleware for calls made inside generated code. " - "Keep FIDES-dependent tools as direct agent tools.", - tool_obj.name, - ) + _warn_unsupported_fides_tools(new_tools, self._fides_warned_metadata) self._refresh_approval_mode() def get_tools(self) -> list[FunctionTool]: @@ -358,6 +385,7 @@ def build_instructions(self, *, tools_visible_to_model: bool) -> str: def create_run_tool(self) -> MontyExecuteCodeTool: """Create a run-scoped snapshot of this execute_code surface.""" return MontyExecuteCodeTool( + _fides_warned_metadata=self._fides_warned_metadata, tools=self.get_tools(), approval_mode=self._default_approval_mode, workspace_root=self._workspace_root, diff --git a/python/packages/monty/tests/monty/test_monty_codeact.py b/python/packages/monty/tests/monty/test_monty_codeact.py index d5f5db71cc2..ae4068b29ab 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact.py +++ b/python/packages/monty/tests/monty/test_monty_codeact.py @@ -1625,3 +1625,57 @@ def limited() -> int: assert payload["exc_type"] == "ToolException" assert "maximum invocation limit" in payload["message"] assert limited.invocation_count == 1 + + +def test_run_snapshot_does_not_repeat_unchanged_fides_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = MontyExecuteCodeTool(tools=annotated_tool) + + caplog.clear() + + with caplog.at_level("WARNING", logger="agent_framework"): + codeact.create_run_tool() + codeact.create_run_tool() + + assert "FIDES is not supported with CodeAct providers." not in caplog.text + + +def test_run_snapshot_warns_when_fides_metadata_keys_change( + caplog: pytest.LogCaptureFixture, +) -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = MontyExecuteCodeTool(tools=annotated_tool) + assert annotated_tool.additional_properties is not None + annotated_tool.additional_properties["confidentiality"] = "private" + + with caplog.at_level("WARNING", logger="agent_framework"): + codeact.create_run_tool() + + assert "FIDES is not supported with CodeAct providers." in caplog.text + assert codeact.get_tools()[0] is annotated_tool + + +def test_run_snapshot_warns_when_fides_metadata_value_changes(caplog: pytest.LogCaptureFixture) -> None: + @tool(additional_properties={"source_integrity": "trusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = MontyExecuteCodeTool(tools=annotated_tool) + assert annotated_tool.additional_properties is not None + + caplog.clear() + annotated_tool.additional_properties["source_integrity"] = "untrusted" + + with caplog.at_level("WARNING", logger="agent_framework"): + run_tool = codeact.create_run_tool() + + assert "FIDES is not supported with CodeAct providers." in caplog.text + assert run_tool.get_tools()[0] is annotated_tool From 1f2381b01b7c7278d6adbb2adeb89cf5e3d72655 Mon Sep 17 00:00:00 2001 From: suhasagg Date: Wed, 28 Oct 2026 16:04:55 +0530 Subject: [PATCH 2/2] fix(codeact): prune FIDES warning cache and deduplicate run snapshots --- .../_execute_code_tool.py | 23 +++++-- .../hyperlight/test_hyperlight_codeact.py | 67 +++++++++++++++++++ .../_execute_code_tool.py | 15 ++++- .../monty/tests/monty/test_monty_codeact.py | 67 +++++++++++++++++++ 4 files changed, 167 insertions(+), 5 deletions(-) diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 802144761af..3c97dcce5a2 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -1659,6 +1659,11 @@ def add_tools( new_tools = _collect_tools(tools) combined_tools = _collect_tools(self._managed_tools, new_tools) self._managed_tools = combined_tools + self._fides_warned_metadata[:] = [ + (registered, metadata) + for registered, metadata in self._fides_warned_metadata + if any(registered is current for current in self._managed_tools) + ] _warn_unsupported_fides_tools(new_tools, self._fides_warned_metadata) self._refresh_approval_mode() @@ -1674,12 +1679,18 @@ def remove_tool(self, name: str) -> None: if len(remaining_tools) == len(self._managed_tools): raise KeyError(f"No managed tool named {name!r} is registered.") self._managed_tools = remaining_tools + self._fides_warned_metadata[:] = [ + (registered, metadata) + for registered, metadata in self._fides_warned_metadata + if any(registered is current for current in remaining_tools) + ] self._refresh_approval_mode() def clear_tools(self) -> None: """Remove all managed sandbox tools.""" with self._state_lock: self._managed_tools = [] + self._fides_warned_metadata.clear() self._refresh_approval_mode() def add_file_mounts(self, file_mounts: FileMountInput | Sequence[FileMountInput]) -> None: @@ -1762,12 +1773,16 @@ def build_instructions(self, *, tools_visible_to_model: bool) -> str: def create_run_tool(self) -> HyperlightExecuteCodeTool: """Create a run-scoped snapshot of this execute_code surface.""" - file_mounts = self.get_file_mounts() - allowed_domains = self.get_allowed_domains() + with self._state_lock: + tools = list(self._managed_tools) + _warn_unsupported_fides_tools(tools, self._fides_warned_metadata) + warned_metadata = list(self._fides_warned_metadata) + file_mounts = self.get_file_mounts() + allowed_domains = self.get_allowed_domains() return HyperlightExecuteCodeTool( - _fides_warned_metadata=self._fides_warned_metadata, - tools=self.get_tools(), + _fides_warned_metadata=warned_metadata, + tools=tools, tool_description_format=self._tool_description_format, approval_mode=self._default_approval_mode, workspace_root=self._workspace_root, diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index dd9c6c61ebd..2e4cfcb1862 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -3555,3 +3555,70 @@ def annotated_tool(value: int) -> int: assert "FIDES is not supported with CodeAct providers." in caplog.text assert run_tool.get_tools()[0] is annotated_tool + + +def test_fides_warning_cache_pruned_on_tool_removal() -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def first(value: int) -> int: + return value + + codeact = HyperlightExecuteCodeTool(tools=first, _registry=_FakeRuntime()) + assert any(obj is first for obj, _ in codeact._fides_warned_metadata) + + codeact.remove_tool(first.name) + + assert not any(obj is first for obj, _ in codeact._fides_warned_metadata) + + +def test_fides_warning_cache_pruned_on_clear() -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def first(value: int) -> int: + return value + + codeact = HyperlightExecuteCodeTool(tools=first, _registry=_FakeRuntime()) + codeact.clear_tools() + + assert codeact._fides_warned_metadata == [] + + +def test_fides_warning_cache_pruned_on_replacement() -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def first(value: int) -> int: + return value + + codeact = HyperlightExecuteCodeTool(tools=first, _registry=_FakeRuntime()) + + @tool( + name=first.name, + additional_properties={"source_integrity": "trusted"}, + ) + def replacement(value: int) -> int: + return value + + codeact.add_tools(replacement) + + assert not any(obj is first for obj, _ in codeact._fides_warned_metadata) + assert any(obj is replacement for obj, _ in codeact._fides_warned_metadata) + + +def test_fides_metadata_change_warns_once_across_run_snapshots( + caplog: pytest.LogCaptureFixture, +) -> None: + @tool(additional_properties={"source_integrity": "trusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = HyperlightExecuteCodeTool(tools=annotated_tool, _registry=_FakeRuntime()) + assert annotated_tool.additional_properties is not None + + annotated_tool.additional_properties["source_integrity"] = "untrusted" + caplog.clear() + + with caplog.at_level("WARNING", logger="agent_framework"): + codeact.create_run_tool() + codeact.create_run_tool() + + warnings = [ + record for record in caplog.records if "FIDES is not supported with CodeAct providers." in record.getMessage() + ] + assert len(warnings) == 1 diff --git a/python/packages/monty/agent_framework_monty/_execute_code_tool.py b/python/packages/monty/agent_framework_monty/_execute_code_tool.py index 46b42798419..5fbc5114b45 100644 --- a/python/packages/monty/agent_framework_monty/_execute_code_tool.py +++ b/python/packages/monty/agent_framework_monty/_execute_code_tool.py @@ -313,6 +313,11 @@ def add_tools( """Add Monty-side tools to this execute_code surface.""" new_tools = _collect_tools(tools) self._managed_tools = _collect_tools(self._managed_tools, new_tools) + self._fides_warned_metadata[:] = [ + (registered, metadata) + for registered, metadata in self._fides_warned_metadata + if any(registered is current for current in self._managed_tools) + ] _warn_unsupported_fides_tools(new_tools, self._fides_warned_metadata) self._refresh_approval_mode() @@ -326,11 +331,17 @@ def remove_tool(self, name: str) -> None: if len(remaining_tools) == len(self._managed_tools): raise KeyError(f"No managed tool named {name!r} is registered.") self._managed_tools = remaining_tools + self._fides_warned_metadata[:] = [ + (registered, metadata) + for registered, metadata in self._fides_warned_metadata + if any(registered is current for current in remaining_tools) + ] self._refresh_approval_mode() def clear_tools(self) -> None: """Remove all managed Monty tools.""" self._managed_tools = [] + self._fides_warned_metadata.clear() self._refresh_approval_mode() def add_file_mounts(self, file_mounts: FileMountInput | Sequence[FileMountInput]) -> None: @@ -384,9 +395,11 @@ def build_instructions(self, *, tools_visible_to_model: bool) -> str: def create_run_tool(self) -> MontyExecuteCodeTool: """Create a run-scoped snapshot of this execute_code surface.""" + tools = self.get_tools() + _warn_unsupported_fides_tools(tools, self._fides_warned_metadata) return MontyExecuteCodeTool( _fides_warned_metadata=self._fides_warned_metadata, - tools=self.get_tools(), + tools=tools, approval_mode=self._default_approval_mode, workspace_root=self._workspace_root, file_mounts=list(self._file_mounts.values()) or None, diff --git a/python/packages/monty/tests/monty/test_monty_codeact.py b/python/packages/monty/tests/monty/test_monty_codeact.py index ae4068b29ab..16bc3b78c0a 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact.py +++ b/python/packages/monty/tests/monty/test_monty_codeact.py @@ -1679,3 +1679,70 @@ def annotated_tool(value: int) -> int: assert "FIDES is not supported with CodeAct providers." in caplog.text assert run_tool.get_tools()[0] is annotated_tool + + +def test_fides_warning_cache_pruned_on_tool_removal() -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def first(value: int) -> int: + return value + + codeact = MontyExecuteCodeTool(tools=first) + assert any(obj is first for obj, _ in codeact._fides_warned_metadata) + + codeact.remove_tool(first.name) + + assert not any(obj is first for obj, _ in codeact._fides_warned_metadata) + + +def test_fides_warning_cache_pruned_on_clear() -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def first(value: int) -> int: + return value + + codeact = MontyExecuteCodeTool(tools=first) + codeact.clear_tools() + + assert codeact._fides_warned_metadata == [] + + +def test_fides_warning_cache_pruned_on_replacement() -> None: + @tool(additional_properties={"source_integrity": "untrusted"}) + def first(value: int) -> int: + return value + + codeact = MontyExecuteCodeTool(tools=first) + + @tool( + name=first.name, + additional_properties={"source_integrity": "trusted"}, + ) + def replacement(value: int) -> int: + return value + + codeact.add_tools(replacement) + + assert not any(obj is first for obj, _ in codeact._fides_warned_metadata) + assert any(obj is replacement for obj, _ in codeact._fides_warned_metadata) + + +def test_fides_metadata_change_warns_once_across_run_snapshots( + caplog: pytest.LogCaptureFixture, +) -> None: + @tool(additional_properties={"source_integrity": "trusted"}) + def annotated_tool(value: int) -> int: + return value + + codeact = MontyExecuteCodeTool(tools=annotated_tool) + assert annotated_tool.additional_properties is not None + + annotated_tool.additional_properties["source_integrity"] = "untrusted" + caplog.clear() + + with caplog.at_level("WARNING", logger="agent_framework"): + codeact.create_run_tool() + codeact.create_run_tool() + + warnings = [ + record for record in caplog.records if "FIDES is not supported with CodeAct providers." in record.getMessage() + ] + assert len(warnings) == 1