fix(templates): map LITELLM_API_KEY to OPENAI_API_KEY in Temporal workers

.stats.yml

@@ -1,4 +1,4 @@
 configured_endpoints: 75
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-7acaeb315af90255109ae17afc71e32a8e5851bb8a956a2a284cb4d344dfab51.yml
-openapi_spec_hash: 3044e94b48d60311b6048e8df88e7552
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-ee0c521f0612c31b874bd595b90cd9209545bab603983552a1a7a87f38ed931e.yml
+openapi_spec_hash: 917a1ffe9e353bed2740524dec786ed2
 config_hash: 593e89b291976a5e84e4c3c3f8324354

src/agentex/lib/adk/__init__.py

@@ -31,6 +31,9 @@
 
 # Data-source refs for lineage (SGP-6513); implementation lives in core.tracing
 from agentex.lib.core.tracing import lineage
+
+# Opt-in commit-SHA stamping (AGX1-969); implementation in core.tracing
+from agentex.lib.core.tracing import code_revision
 from agentex.lib.core.tracing.lineage import DataSourceRef, data_sources
 
 # Unified harness surface (AGX1-375)
@@ -73,6 +76,7 @@
     "TurnSpan",
     # Lineage data-source refs (SGP-6513)
     "lineage",
+    "code_revision",
     "DataSourceRef",
     "data_sources",
     # Checkpointing / LangGraph

src/agentex/lib/cli/debug/debug_handlers.py

@@ -16,6 +16,7 @@
     pass
 
 from agentex.lib.utils.logging import make_logger
+from agentex.lib.cli.utils.cli_utils import SUBPROCESS_STREAM_LIMIT
 
 from .debug_config import DebugConfig, resolve_debug_port
 
@@ -66,6 +67,7 @@ async def start_temporal_worker_debug(
         env=debug_env,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.STDOUT,
+        limit=SUBPROCESS_STREAM_LIMIT,
     )
 
 
@@ -119,6 +121,7 @@ async def start_acp_server_debug(
         env=debug_env,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.STDOUT,
+        limit=SUBPROCESS_STREAM_LIMIT,
     )
 
 

src/agentex/lib/cli/handlers/run_handlers.py

@@ -12,6 +12,7 @@
 from agentex.lib.cli.debug import DebugConfig, start_acp_server_debug, start_temporal_worker_debug
 from agentex.lib.utils.logging import make_logger
 from agentex.config.agent_manifest import AgentManifest
+from agentex.lib.cli.utils.cli_utils import SUBPROCESS_STREAM_LIMIT
 from agentex.lib.cli.utils.path_utils import (
     get_file_paths,
     calculate_uvicorn_target_for_local,
@@ -23,6 +24,11 @@
 logger = make_logger(__name__)
 console = Console()
 
+# How many consecutive unreadable lines to skip before giving up on the stream.
+# Skipping is only known-safe for the limit-overrun case; this bounds the damage
+# if some other error repeats without consuming anything.
+MAX_CONSECUTIVE_READ_ERRORS = 100
+
 
 class RunError(Exception):
     """An error occurred during agent run"""
@@ -215,6 +221,7 @@ async def start_acp_server(
         env=env,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.STDOUT,
+        limit=SUBPROCESS_STREAM_LIMIT,
     )
 
 
@@ -234,23 +241,68 @@ async def start_temporal_worker(
         env=env,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.STDOUT,
+        limit=SUBPROCESS_STREAM_LIMIT,
     )
 
 
 async def stream_process_output(process: asyncio.subprocess.Process, prefix: str):
-    """Stream process output with prefix"""
+    """Stream process output with prefix.
+
+    This loop is the only reader of the child's stdout pipe. If it ever stops
+    reading, the pipe fills and the child blocks forever inside ``write()``,
+    which presents as a silent freeze: 0% CPU, no further logs, no traceback.
+    So a single unreadable line must never end the loop.
+    """
     try:
         if process.stdout is None:
             return
+        consecutive_read_errors = 0
         while True:
-            line = await process.stdout.readline()
+            try:
+                line = await process.stdout.readline()
+            except ValueError as e:
+                # readline() raises ValueError when a line exceeds the stream limit.
+                # In *that* case it has already discarded the line and resumed the
+                # transport, so skipping it makes guaranteed progress. Any other
+                # ValueError carries no such guarantee, and retrying it forever would
+                # spin without draining. We cannot tell the two apart (readline
+                # flattens LimitOverrunError into a bare ValueError), so bound the
+                # retries and let the outer handler report the hang risk.
+                consecutive_read_errors += 1
+                if consecutive_read_errors > MAX_CONSECUTIVE_READ_ERRORS:
+                    raise
+                logger.warning(
+                    f"Skipping an unreadable line from {prefix}: {e!r} "
+                    f"(consecutive failure {consecutive_read_errors}/{MAX_CONSECUTIVE_READ_ERRORS}). "
+                    f"If this says the chunk exceeded the limit, raise limit= on this "
+                    f"process's create_subprocess_exec."
+                )
+                continue
+
+            consecutive_read_errors = 0
+
             if not line:
                 break
-            decoded_line = line.decode("utf-8").rstrip()
+
+            try:
+                decoded_line = line.decode("utf-8").rstrip()
+            except UnicodeDecodeError as e:
+                logger.warning(f"Dropped an undecodable log line from {prefix} ({e}).")
+                continue
+
             if decoded_line:  # Only print non-empty lines
                 console.print(f"[dim]{prefix}:[/dim] {decoded_line}")
     except Exception as e:
-        logger.debug(f"Output streaming ended for {prefix}: {e}")
+        # The escalation path, including for the re-raise above. Anything reaching
+        # here ends the loop, so the child is now at risk of blocking on a full pipe.
+        # Warning rather than debug: this used to be a debug() that make_logger could
+        # never emit, which is why three freezes produced no clue.
+        # CancelledError derives from BaseException, so the auto-reload path that
+        # cancels these tasks passes straight through and is unaffected.
+        logger.warning(
+            f"Output streaming for {prefix} stopped on {e!r}. "
+            f"Nothing is draining its stdout now, so {prefix} will hang once the pipe fills."
+        )
 
 
 async def run_agent(manifest_path: str, debug_config: "DebugConfig | None" = None):

src/agentex/lib/cli/templates/temporal-openai-agents/project/run_worker.py.j2

@@ -1,3 +1,18 @@
+import os
+
+# Load the project .env first (the ACP process does the same), then map the
+# LiteLLM proxy key: the worker process runs the model calls, so copy
+# LITELLM_API_KEY to OPENAI_API_KEY before importing project code (some
+# frameworks build their OpenAI client at import time). Without this the
+# worker raises "Missing credentials".
+from dotenv import load_dotenv
+
+load_dotenv()  # the project .env, before any project code runs
+
+_litellm_key = os.environ.get("LITELLM_API_KEY")
+if _litellm_key and not os.environ.get("OPENAI_API_KEY"):
+    os.environ["OPENAI_API_KEY"] = _litellm_key
+
 import asyncio
 
 from agentex.lib.core.temporal.activities import get_all_activities
@@ -15,6 +30,7 @@ from agentex.lib.core.temporal.plugins.openai_agents.models.temporal_streaming_m
 from project.workflow import {{ workflow_class }}
 from project.activities import get_weather
 
+
 environment_variables = EnvironmentVariables.refresh()
 
 logger = make_logger(__name__)

src/agentex/lib/cli/templates/temporal-pydantic-ai/project/run_worker.py.j2

@@ -8,6 +8,22 @@ class and registers every model/tool activity the TemporalAgent needs —
 so we don't have to enumerate activities by hand here.
 """
 
+import os
+
+# Load the project .env first (the ACP process does the same), then map the
+# LiteLLM proxy key: the worker process runs the model calls, so copy
+# LITELLM_API_KEY to OPENAI_API_KEY before importing project code (some
+# frameworks build their OpenAI client at import time). Without this the
+# worker raises "Missing credentials".
+from dotenv import load_dotenv
+
+load_dotenv()  # the project .env, before any project code runs
+
+_litellm_key = os.environ.get("LITELLM_API_KEY")
+if _litellm_key and not os.environ.get("OPENAI_API_KEY"):
+    os.environ["OPENAI_API_KEY"] = _litellm_key
+
+
 import asyncio
 
 from project.workflow import {{ workflow_class }}
@@ -19,6 +35,7 @@ from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.activities import get_all_activities
 from agentex.lib.core.temporal.workers.worker import AgentexWorker
 
+
 environment_variables = EnvironmentVariables.refresh()
 logger = make_logger(__name__)
 

src/agentex/lib/cli/utils/cli_utils.py

@@ -5,6 +5,18 @@
 
 console = Console()
 
+# asyncio's StreamReader defaults to 64 KiB, and a single log line above that makes
+# readline() raise. Agents legitimately emit large lines (serialized charts, payloads
+# echoed back by validation errors), so give the reader room before it has to drop one.
+#
+# Lives here rather than beside its users so that both the normal spawns in
+# cli/handlers/run_handlers.py and the debug spawns in cli/debug/debug_handlers.py can
+# import it: run_handlers imports cli.debug, so the constant cannot live in either one.
+# Keep the two in step. A subprocess left on the asyncio default overruns far more
+# easily, and enough consecutive overruns exhaust the reader's retry bound and stop it
+# draining, which is the deadlock the bound is there to avoid.
+SUBPROCESS_STREAM_LIMIT = 8 * 1024 * 1024
+
 
 def handle_questionary_cancellation(
     result: str | None, operation: str = "operation"

src/agentex/lib/core/tracing/code_revision.py

@@ -0,0 +1,105 @@
+"""Opt-in stamping of the agent's source commit onto its spans.
+
+Nothing is stamped until the agent calls :func:`enable`, mirroring the
+``lineage`` registry next door: a process-wide switch the agent sets once at
+import, rather than automatic behaviour every agent inherits. When enabled the
+resolved commit lands in span data under ``__commit_sha__`` and is searchable in
+the SGP Traces UI as ``__commit_sha__:<sha>``.
+
+This is deliberately separate from ``__agent_version__``, which is automatic and
+carries the deployed image tag verbatim ("image tag or git sha"). That tag is a
+real commit on some build paths but an ``<image-name>-<sha>`` composite (AWS
+ECR), ``latest``, or a hand-passed tag on others -- so a field named for a commit
+must not simply mirror it. Values that are not git object names are refused, and
+a field named ``__commit_sha__`` therefore only ever holds one.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+from agentex.lib.utils.logging import make_logger
+
+__all__ = ("COMMIT_SHA_KEY", "enable", "disable", "is_enabled", "commit_sha")
+
+logger = make_logger(__name__)
+
+COMMIT_SHA_KEY = "__commit_sha__"
+
+# A git object name: 40 hex for SHA-1, 64 for SHA-256, or an abbreviation down to
+# git's own 7-character minimum.
+_GIT_SHA_RE = re.compile(r"[0-9a-fA-F]{7,64}")
+
+_COMMIT_SHA_ENV = "AGENT_COMMIT_SHA"
+# Fallback only: automatic, and only usable when it happens to be SHA-shaped.
+_AGENT_VERSION_ENV = "AGENT_VERSION"
+
+# Resolved once at enable() rather than per span: the value is fixed for the
+# life of the process, and resolving eagerly means a bad value is reported at
+# startup instead of silently producing unstamped spans.
+_commit_sha: str | None = None
+
+
+def enable(commit_sha: str | None = None) -> None:
+    """Opt this process in to stamping ``__commit_sha__`` onto every span.
+
+    Value precedence: the explicit ``commit_sha`` argument, else
+    ``AGENT_COMMIT_SHA``, else ``AGENT_VERSION`` when the deployment happened to
+    set it to a bare commit SHA. A value that is not a git object name is
+    refused with a warning and leaves stamping off -- better an absent field
+    than one named for a commit that holds an image tag.
+    """
+    global _commit_sha
+
+    for value, source in (
+        (commit_sha, "the commit_sha argument"),
+        (os.environ.get(_COMMIT_SHA_ENV), _COMMIT_SHA_ENV),
+        (os.environ.get(_AGENT_VERSION_ENV), _AGENT_VERSION_ENV),
+    ):
+        candidate = (value or "").strip()
+        if not candidate:
+            continue
+        if _GIT_SHA_RE.fullmatch(candidate):
+            _commit_sha = candidate
+            logger.info("code revision stamping enabled from %s", source)
+            return
+        # An explicit argument or AGENT_COMMIT_SHA is a direct statement of
+        # intent, so a bad value there is worth surfacing. AGENT_VERSION is only
+        # a fallback and is expected to be a non-SHA tag much of the time, so
+        # falling through it quietly is correct, not a silent failure.
+        if source != _AGENT_VERSION_ENV:
+            logger.warning(
+                "%s=%r is not a git commit SHA; __commit_sha__ will not be stamped.",
+                source,
+                candidate,
+            )
+            _commit_sha = None
+            return
+
+    _commit_sha = None
+    logger.warning(
+        "code revision stamping was enabled but no commit SHA was found "
+        "(checked the commit_sha argument, %s, and %s); __commit_sha__ will not "
+        "be stamped. Set %s in the agent's environment -- e.g. bake it at build "
+        "time with a Dockerfile ARG/ENV.",
+        _COMMIT_SHA_ENV,
+        _AGENT_VERSION_ENV,
+        _COMMIT_SHA_ENV,
+    )
+
+
+def disable() -> None:
+    """Turn stamping back off (also used for test isolation)."""
+    global _commit_sha
+    _commit_sha = None
+
+
+def is_enabled() -> bool:
+    """Whether a commit SHA resolved and will be stamped."""
+    return _commit_sha is not None
+
+
+def commit_sha() -> str | None:
+    """The resolved commit SHA, or ``None`` when stamping is not enabled."""
+    return _commit_sha

src/agentex/lib/core/tracing/processors/sgp_tracing_processor.py

@@ -3,14 +3,15 @@
 import os
 import asyncio
 import weakref
-from typing import cast, override
+from typing import Any, cast, override
 
 import scale_gp_beta.lib.tracing as tracing
 from scale_gp_beta import SGPClient, AsyncSGPClient
 from scale_gp_beta.lib.tracing import create_span, flush_queue
 from scale_gp_beta.lib.tracing.span import Span as SGPSpan
 
 from agentex.types.span import Span
+from agentex.lib.core.tracing import code_revision
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
 from agentex.lib.utils.logging import make_logger
 from agentex.lib.core.observability import tracing_metrics_recording as _metrics
@@ -69,6 +70,29 @@ def _add_source_to_span(span: Span, env_vars: EnvironmentVariables) -> None:
             span.data["__agent_version__"] = env_vars.AGENT_VERSION
 
 
+def _sgp_metadata(span: Span) -> Any:
+    """Metadata for the SGP write: ``span.data`` plus the opt-in commit SHA.
+
+    Returns a COPY rather than mutating ``span``. ``trace.py`` hands the same
+    Span instance to every registered processor, so anything written onto
+    ``span.data`` here would also be serialized by the Agentex processor and
+    show up in caller-visible span data. ``__commit_sha__`` is opt-in and
+    SGP-scoped, so it must not leak that way.
+
+    (The ``__source__`` / ``__agent_*`` keys set by ``_add_source_to_span`` do
+    leak like that today. Left as-is: changing five long-shipped fields is not
+    this change's business.)
+    """
+    commit_sha = code_revision.commit_sha()
+    if commit_sha is None:
+        return span.data
+    if isinstance(span.data, dict):
+        return {**span.data, code_revision.COMMIT_SHA_KEY: commit_sha}
+    # List-shaped data is an accepted `data` shape and has nowhere to put a
+    # metadata key; leave it untouched rather than dropping the caller's data.
+    return span.data
+
+
 def _build_sgp_span(span: Span, env_vars: EnvironmentVariables) -> SGPSpan:
     """Build an SGPSpan from an agentex Span. Idempotent on span_id at the SGP backend."""
     _add_source_to_span(span, env_vars)
@@ -82,7 +106,7 @@ def _build_sgp_span(span: Span, env_vars: EnvironmentVariables) -> SGPSpan:
             trace_id=span.trace_id,
             input=span.input,
             output=span.output,
-            metadata=span.data,
+            metadata=_sgp_metadata(span),
         ),
     )
     sgp_span.start_time = span.start_time.isoformat()  # type: ignore[union-attr]