From 3de63cce6a7b3e57fe6ce5cef51543dafd5ec4b7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 08:56:02 -0800 Subject: [PATCH 1/2] fix(prismml): use released model metadata and test current SDK Refresh published recipe pins, require model attribution on streams and errors, test SDK bounds, and replace verification prose with executable setup guidance. Co-Authored-By: Claude Code --- .github/workflows/prismml-recipe-tests.yml | 24 +- python/examples/prismml/README.md | 228 ++++++------------ python/examples/prismml/requirements.txt | 8 +- .../prismml/tests/test_prismml_recipe.py | 85 ++++--- 4 files changed, 159 insertions(+), 186 deletions(-) diff --git a/.github/workflows/prismml-recipe-tests.yml b/.github/workflows/prismml-recipe-tests.yml index a7ca8deaf..d06063e76 100644 --- a/.github/workflows/prismml-recipe-tests.yml +++ b/.github/workflows/prismml-recipe-tests.yml @@ -16,13 +16,25 @@ permissions: jobs: recipe: - name: PrismML recipe (${{ matrix.packages }}) + name: PrismML recipe (${{ matrix.packages }}, Python ${{ matrix.python }}, OpenAI ${{ matrix.openai }}) runs-on: ubuntu-latest timeout-minutes: 10 strategy: fail-fast: false matrix: - packages: [published, source] + include: + - packages: published + python: '3.10' + openai: '1.69.0' + - packages: source + python: '3.10' + openai: '1.69.0' + - packages: published + python: '3.13' + openai: '3.42.1' + - packages: source + python: '3.13' + openai: '3.42.1' env: FI_BASE_URL: http://127.0.0.1:9 FI_GRPC_URL: http://127.0.0.1:9 @@ -35,11 +47,15 @@ jobs: persist-credentials: false - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: ${{ matrix.python }} - name: Install published recipe pins and test tools - run: pip install -r python/examples/prismml/requirements.txt pytest httpx 'wrapt<2' + run: pip install -r python/examples/prismml/requirements.txt pytest httpx 'wrapt<2' PyYAML + - name: Select SDK compatibility version + run: pip install 'openai==${{ matrix.openai }}' - name: Install source distributions if: matrix.packages == 'source' run: pip install ./python ./python/frameworks/openai + - name: Check dependency consistency + run: pip check - name: Loopback-only recipe contracts run: pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs diff --git a/python/examples/prismml/README.md b/python/examples/prismml/README.md index 6a9470ada..c273054dd 100644 --- a/python/examples/prismml/README.md +++ b/python/examples/prismml/README.md @@ -1,203 +1,131 @@ # PrismML (OpenAI-compatible) with traceAI -PrismML here means the Bonsai local server from prismml.com; this recipe traces Chat Completions through the `openai` SDK at `http://localhost:8080/v1` ([server docs](https://docs.prismml.com/run/server)). It uses `traceai-openai`, with no PrismML instrumentation package. - -Not these: other products named Prism are not covered here, for example the [ssimplifi Prism proxy guide](https://ssimplifi.com/guides/openai-compatible-api), [prism-proxy](https://git.sovereign-society.org/prism/prism-proxy) and [Prism PHP](https://mintlify.wiki/prism-php/prism/providers/overview). +Trace calls to your local [PrismML Bonsai server](https://docs.prismml.com/run/server) in Future AGI using the maintained `traceai-openai` package. This recipe uses the OpenAI Python SDK's Chat Completions API; it is not an integration for other products named Prism. ## Install -From this recipe directory, install the pinned dependencies: +Use Python 3.10–3.13. From this recipe directory: + +```bash +python -m pip install -r requirements.txt +``` + +For an existing application, install the same packages directly: ```bash -pip install -r requirements.txt +python -m pip install 'openai==3.42.1' 'traceAI-openai==0.1.12' 'fi-instrumentation-otel==1.2.2' ``` -The plain installation form is: +## Start the local server + +Follow the [PrismML server setup](https://docs.prismml.com/run/server) to download a model and start the llama.cpp server. In the Bonsai demo directory, give the loaded model a stable name: ```bash -pip install traceAI-openai fi-instrumentation-otel openai +./scripts/start_llama_server.sh --alias bonsai ``` -The test versions and verification scope are listed below. +The API base URL is `http://localhost:8080/v1`. Keep the server on loopback: it does not authenticate requests. Binding to a non-loopback address exposes it to your network; use a trusted network or firewall the port. + +On Apple Silicon, PrismML also provides `./scripts/start_mlx_server.sh` at `http://localhost:8081/v1`. Follow its [sampling and thinking configuration](https://docs.prismml.com/run/server); unlike the llama.cpp launcher, the MLX server expects those settings in each request. ## Configure -Start your server first, following the [PrismML server docs](https://docs.prismml.com/run/server): `./scripts/start_llama_server.sh` serves llama.cpp on port 8080; `./scripts/start_mlx_server.sh` serves MLX on Apple Silicon on port 8081. Both expose the same OpenAI-compatible API. +Set your Future AGI credentials in the shell that runs the Python client: -| Variable | Value or default | Destination | -|---|---|---| -| `FI_API_KEY` | Your Future AGI API key | Tracer only; never the local server | -| `FI_SECRET_KEY` | Your Future AGI secret key | Tracer only; never the local server | -| `PRISMML_API_KEY` | Optional; `not-needed` | OpenAI client only; never the Future AGI export | -| `PRISMML_MODEL` | Optional; `bonsai` | Model name sent by the OpenAI client | -| `PRISMML_BASE_URL` | Optional; `http://localhost:8080/v1` | OpenAI client; use `http://localhost:8081/v1` for MLX | -| `FI_BASE_URL` | Optional; the tracer's Future AGI collector default | Tracer export origin | -| `FI_PROJECT_NAME` | Optional; `prismml-bonsai` | Tracer only; the Observe project name | -| `FI_HIDE_INPUTS` | Optional; `true` masks input attributes | Tracer only | -| `FI_HIDE_OUTPUTS` | Optional; `true` masks output attributes | Tracer only | -| `FI_HIDE_LLM_INVOCATION_PARAMETERS` | Optional; `true` omits invocation parameters | Tracer only | +```bash +export FI_API_KEY="YOUR_FUTURE_AGI_API_KEY" +export FI_SECRET_KEY="YOUR_FUTURE_AGI_SECRET_KEY" +export FI_PROJECT_NAME="prismml-bonsai" +export PRISMML_BASE_URL="http://localhost:8080/v1" +export PRISMML_MODEL="bonsai" +``` -The local key is a placeholder, not a Future AGI secret. The OpenAI SDK requires a non-empty key, but the server does not check it ([server docs](https://docs.prismml.com/run/server)). An unset or empty `PRISMML_API_KEY` uses `not-needed`; an unset or empty `PRISMML_MODEL` uses `bonsai`. +The tracer sends OTLP to `https://api.futureagi.com/tracer/v1/traces` by default. The OpenAI client sends model requests only to `PRISMML_BASE_URL`. Do not use your Future AGI credentials as the local server key. -The recipe validates `PRISMML_BASE_URL` before tracing. Invalid URLs exit 2 and name that variable. An explicitly empty `--model` exits 2 and names `PRISMML_MODEL`. The recipe does not validate the Future AGI keys. +| Variable | Value or default | Purpose | +|---|---|---| +| `FI_API_KEY` | Your Future AGI API key | Authenticate trace export | +| `FI_SECRET_KEY` | Your Future AGI secret key | Authenticate trace export | +| `FI_PROJECT_NAME` | Optional; `prismml-bonsai` | Observe project name | +| `FI_BASE_URL` | Optional; `https://api.futureagi.com` | Trace export origin; omit `/tracer/v1/traces` | +| `PRISMML_BASE_URL` | Optional; `http://localhost:8080/v1` | Local server API base URL | +| `PRISMML_MODEL` | Optional; `bonsai` | Requested model name | +| `PRISMML_API_KEY` | Optional; `not-needed` | Non-empty placeholder required by the OpenAI SDK | ## Run -From this recipe directory, with your server already running: +Return to this recipe directory and run a normal or streamed completion: ```bash -export FI_API_KEY="placeholder-fi-api-key" -export FI_SECRET_KEY="placeholder-fi-secret-key" -export PRISMML_API_KEY="not-needed" -export PRISMML_MODEL="bonsai" -export PRISMML_BASE_URL="http://localhost:8080/v1" -python src/app.py -python src/app.py --stream +python src/app.py --prompt "What is the capital of France?" +python src/app.py --stream --prompt "What is the capital of France?" ``` -Replace the Future AGI placeholders with your keys. `--prompt` changes the question and `--model` overrides `PRISMML_MODEL`. Set `FI_PROJECT_NAME` to send the traces to another Observe project; unset or empty, they go to `prismml-bonsai`. The app prints answer text and flushes the tracer. +The application prints the answer and flushes traces before exiting. `--model` overrides `PRISMML_MODEL`. An unset or empty project name uses `prismml-bonsai`. ## Code -Instrument before creating the client. Future AGI keys are read by the tracer from its environment. +For your own application, register the tracer and instrument the SDK once, before creating the client. Do not add a second OpenAI instrumentor around the same requests. ```python import os + from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType from openai import OpenAI from traceai_openai import OpenAIInstrumentor -provider = register( # FI_PROJECT_NAME overrides, as in src/app.py - project_name=os.environ.get("FI_PROJECT_NAME") or "prismml-bonsai", project_type=ProjectType.OBSERVE, - set_global_tracer_provider=False, verbose=False, + +provider = register( + project_name=os.environ.get("FI_PROJECT_NAME") or "prismml-bonsai", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, ) OpenAIInstrumentor().instrument(tracer_provider=provider) -with OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed") as client: - response = client.chat.completions.create( - model="bonsai", messages=[{"role": "user", "content": "Say hello."}], stream=False, - ) - print(response.choices[0].message.content or "") -provider.force_flush() +try: + with OpenAI( + base_url=os.environ.get("PRISMML_BASE_URL") or "http://localhost:8080/v1", + api_key=os.environ.get("PRISMML_API_KEY") or "not-needed", + ) as client: + response = client.chat.completions.create( + model=os.environ.get("PRISMML_MODEL") or "bonsai", + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + print(response.choices[0].message.content or "") +finally: + provider.force_flush() ``` -## What you see in Future AGI - -The fixtures produce one `ChatCompletion` span per call with `gen_ai.span.kind=LLM`. The provider field says `openai`: `gen_ai.provider.name` is the shared instrumentor's label. - -Current `traceai-openai` behavior: - -- For a successful non-streamed call, `gen_ai.request.model` is the model name the server returns. The fixture requests `bonsai` and returns the synthetic loaded-model name `Ternary-Bonsai-2-27B-Q4_K_M.gguf`; the latter is recorded. The requested `bonsai` remains in the `gen_ai.request.parameters` JSON. A real `llama-server` started by the demo scripts returns the loaded file's absolute path, such as `/Users/you/Bonsai-demo/models/gguf/1.7B/Bonsai-1.7B-Q1_0.gguf`, and that path is what gets recorded. Start the server with `./scripts/start_llama_server.sh --alias bonsai` to record `bonsai` instead; the script passes extra arguments to `llama-server`. -- Up to `traceai-openai` 0.1.10, streamed and failed calls omit `gen_ai.request.model`; later versions (future-agi/traceAI#249) record the requested model there. The requested model remains in `gen_ai.request.parameters`, unless invocation parameters are hidden. -- Non-streamed token usage is recorded as `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, and `gen_ai.usage.total_tokens` when the response supplies it. Missing usage is omitted, never replaced with zero. -- A stream accumulates answer text in `output.value`. Default fixture streams omit usage. With `stream_options={"include_usage": True}` and a final usage chunk, the same three token attributes are recorded. This verifies the instrumentor, not the real server's streaming usage support. -- Non-streamed `output.value` contains assistant content. Tool-call responses contain a readable function call instead. A stream with no text records `output.value=''`; Chat Completions have no raw-response fallback here. -- SDK errors set span status to ERROR and record an exception event. The 401 fixture is synthetic; it does not imply that this unauthenticated server checks a key. - -A cost lookup keyed on the requested name may miss the server's returned name. Cost may be unknown when usage or a matching price entry is unavailable. This recipe adds no model pricing. - -## Provider specifics - -According to the [PrismML server docs](https://docs.prismml.com/run/server): - -| Server | API base URL | -|---|---| -| llama.cpp | `http://localhost:8080/v1` | -| MLX on Apple Silicon | `http://localhost:8081/v1` | +## View traces -The SDK appends a trailing slash to its stored base URL and calls `/v1/chat/completions`. The validator returns allowed URLs unchanged. It permits exactly `/v1` or `/v1/` on any host and refuses the root web chat, a full `/v1/chat/completions` URL, other paths, credentials, query or fragment delimiters, whitespace, control characters, non-ASCII hosts and malformed IDNA. Host names may use letters, digits, hyphens and underscores inside a label, so container or Compose service names such as `bonsai_server` work. +Open **Observe** in Future AGI and select the project named by `FI_PROJECT_NAME`. Each completion produces one `ChatCompletion` LLM span with the prompt, answer, latency and any token usage supplied by the server. The provider field says `openai` because this local endpoint uses the shared OpenAI instrumentor. -Loopback HTTP is allowed without a warning for `localhost` and loopback IP addresses (`127.0.0.0/8`, `::1`). HTTP outside loopback is allowed with one stderr warning about the unauthenticated server. Use it only on a trusted LAN. HTTPS URLs still need the same API path. +- Non-streamed calls record the server's returned model in `gen_ai.request.model`. Streamed and failed calls record the requested model. The original requested name also appears in `gen_ai.request.parameters` unless invocation parameters are hidden. +- Streamed answer text is accumulated in `output.value`. To request streaming token counts, pass `stream_options={"include_usage": True}`; counts appear only if the server returns a final usage chunk. Missing usage is omitted, not replaced with zero. +- SDK errors mark the span as ERROR. Returned `tool_calls` are recorded, but the instrumentor does not execute tools. +- Local models may have no matching price entry, so cost can be unavailable even when token counts are present. -The [server docs](https://docs.prismml.com/run/server) warn: - -> The scripts bind to `127.0.0.1`, so the server is reachable only from the same machine and there is no authentication. Setting `BONSAI_HOST` to any non-loopback address (for example `0.0.0.0`) exposes an unauthenticated server to your network. - -For llama-server, any model string works because it serves the loaded model; `GET /v1/models` returns the real name: the loaded file's absolute path by default, or the name set with `--alias`. The documented Python example requests `bonsai` with streaming ([server docs](https://docs.prismml.com/run/server)). `traceai-openai` records the `model` field of the server's non-streamed response, pinned by the tests; the loaded-model name used in tests is synthetic, not a promised filename. - -Thinking is controlled per request with `thinking_budget_tokens`; zero disables it ([server docs](https://docs.prismml.com/run/server)). Send it through the SDK's `extra_body`: - -```python -response = client.chat.completions.create( - model="bonsai", messages=[{"role": "user", "content": "Say hello."}], - extra_body={"thinking_budget_tokens": 0}, -) -``` - -The tests pin that this value reaches the HTTP request but is absent from `gen_ai.request.parameters`. The SDK merges `extra_body` after the instrumentor reads the request parameters. - -The 27B model supports native OpenAI-style `tool_calls` ([server docs](https://docs.prismml.com/run/server)). The fixture uses a standard `tools` array. It records definitions in `gen_ai.tool.definitions` and `gen_ai.tool.definitions.0.tool.json_schema`, and records the returned tool ID, function name and arguments under `gen_ai.output.messages.0.message.tool_calls.0.tool_call.*`. It does not execute a tool. +For llama.cpp thinking control, pass `extra_body={"thinking_budget_tokens": 0}` to `client.chat.completions.create()` to disable thinking for that request. This server-specific option reaches the model server but is not included in the pinned instrumentor's invocation parameters. See the [PrismML server reference](https://docs.prismml.com/run/server) for model-specific settings. ## Privacy -Prompt and answer text normally leave the machine in the Future AGI trace export. Set `FI_HIDE_INPUTS=true` to mask input attributes and `FI_HIDE_OUTPUTS=true` to mask output attributes. The tests verify each flag against visible controls for both chat and streaming. - -The server's returned model name is exported too, and these flags do not mask it. Without `--alias`, `llama-server` returns the model file's absolute path, which can include your user name or directory layout. Start the server with `--alias bonsai` to keep local paths out of the export. - -`FI_HIDE_INPUTS` masks only inputs. If the server echoes the prompt in its answer, that text remains in output attributes unless outputs are also hidden. Non-streamed output is assistant content, and streamed output is accumulated text, including an empty string when no text arrives. This recipe does not export a raw Chat Completions response in `output.value`. - -The local server still receives the prompt. Future AGI masking does not change the server's memory or its own logging. Key separation tests verify that `not-needed` and the custom local placeholder never reach the Future AGI export, while Future AGI keys never reach the local request. - -## Limits / not covered - -- The built-in web chat, Open WebUI, MCP, Responses API, embeddings and other API surfaces. -- Starting or stopping the server, distributing weights, live GPU jobs, MLX server differences, benchmarks or model quality. -- Tool execution and streaming tool calls. -- Other products called Prism: the [ssimplifi Prism proxy guide](https://ssimplifi.com/guides/openai-compatible-api), [prism-proxy](https://git.sovereign-society.org/prism/prism-proxy), and [Prism PHP](https://mintlify.wiki/prism-php/prism/providers/overview). -- The automated contract suite uses a local fake of the OpenAI API and does not download weights. The real-server check below is separate. - -## Real-server check - -On 2026-10-28 this recipe ran against the official PrismML `llama-server` -release `prism-b10826-c9c34bc`, with the official -`Ternary-Bonsai-2-2B-PQ2_0.gguf` weights (verified against the published SHA256SUMS). -The Linux CPU server used `BONSAI_MODEL=2B`, `BONSAI_NGL=0`, `BONSAI_CTX=4096`, -`BONSAI_THINKING=0`, and `--alias bonsai`. The model and binary came from the -[official demo](https://github.com/PrismML-Eng/Bonsai-demo) and its linked releases. - -With the pinned `openai==3.24.0`, `traceAI-openai==0.1.10`, and -`fi-instrumentation-otel==1.1.0`, both the normal and `--stream` recipe commands -returned answers. Their exported spans were read back in Future AGI Cloud: -non-streamed calls had model `bonsai` and token usage; the default stream had -answer text but no model or usage attributes, matching the published-version -behavior described above. A separate call with input/output/invocation-parameter -hiding enabled exported no prompt or answer text. This check covers the 2B text -model on the llama.cpp server, not 27B tool calling, vision, or MLX. - -## Tests - -Run these commands from the repository root. All provider responses are synthetic: in-process MockTransport at both documented URLs, plus a server and OTLP receiver bound to `127.0.0.1`. A subprocess socket guard refuses non-loopback connections before DNS and has a loopback positive control. - -Repository source: +Model execution is local, but prompt and answer content are exported to Future AGI by default. To hide content and invocation parameters, set these variables before running the application: ```bash -env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ - PYTHONPATH="python/examples/prismml/src:python:python/frameworks/openai:python/tests" \ - uv run --no-project --python 3.11 \ - --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ - --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ - --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ - --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ - pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +export FI_HIDE_INPUTS=true +export FI_HIDE_OUTPUTS=true +export FI_HIDE_LLM_INVOCATION_PARAMETERS=true ``` -Published packages: +Hide both input and output if an answer might echo a sensitive prompt. These settings do not change what the local server receives or logs. -```bash -env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ - PYTHONPATH="python/examples/prismml/src:python/tests" \ - uv run --no-project --python 3.11 \ - --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ - --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ - --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ - --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ - --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ - pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs -``` +The returned model name is metadata and is not masked by these flags. Without an alias, llama-server can return the loaded model file's **absolute path**, including your user name or directory layout. Start it with `--alias bonsai` to keep local paths out of traces. + +## Troubleshooting -To check the `traceai-openai` SDK floor, change the source command's OpenAI pin to `openai==1.69.0`. +- **Connection refused:** start the local server first and check the port. Use port 8080 for llama.cpp or 8081 for MLX. +- **URL validation error:** the recipe requires a base URL ending in `/v1` or `/v1/`, not the web chat root or `/v1/chat/completions`. Remove credentials, query strings and fragments. Compose host names such as `bonsai_server` are accepted; non-loopback HTTP prints a security warning. +- **No traces:** check `FI_API_KEY`, `FI_SECRET_KEY` and the project name. `FI_BASE_URL` is the trace export origin, not the model server URL. +- **Unexpected model path or missing cost:** use `--alias bonsai`; pricing still requires a matching model price entry. -| Python | `openai` | traceAI packages | Result | -|---|---|---|---| -| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | -| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | -| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | +This route traces calls made by your Python client. It does not automatically trace the server's built-in web chat or other applications connected to it. diff --git a/python/examples/prismml/requirements.txt b/python/examples/prismml/requirements.txt index 2e789e40c..5ede74902 100644 --- a/python/examples/prismml/requirements.txt +++ b/python/examples/prismml/requirements.txt @@ -1,4 +1,4 @@ -# Installation pins; see README for source verification and the SDK floor. -openai==3.24.0 -traceAI-openai==0.1.10 -fi-instrumentation-otel==1.1.0 +# Published dependencies for the local Bonsai server recipe. +openai==3.42.1 +traceAI-openai==0.1.12 +fi-instrumentation-otel==1.2.2 diff --git a/python/examples/prismml/tests/test_prismml_recipe.py b/python/examples/prismml/tests/test_prismml_recipe.py index 29be3876a..02c89e7b5 100644 --- a/python/examples/prismml/tests/test_prismml_recipe.py +++ b/python/examples/prismml/tests/test_prismml_recipe.py @@ -25,10 +25,6 @@ FakeOpenAI, chat_response, stream_body, ) -# traceai-openai 0.1.10 and earlier omit gen_ai.request.model on streamed and failed -# calls; later versions (future-agi/traceAI#249) record the requested model there. -# Accept both so this recipe passes on either version. - ROOT = Path(__file__).resolve().parents[1] FI_API_KEY = "placeholder-fi-api-key" FI_SECRET_KEY = "placeholder-fi-secret-key" @@ -267,9 +263,7 @@ def test_stream_accumulation_and_usage(traced, include_usage): assert body.get("stream_options") == ({"include_usage": True} if include_usage else None) span, attrs = one_span(traced) assert attrs["output.value"] == ANSWER - # traceai-openai records the model only from a non-streamed response; flip - # this when the instrumentor records the request model. - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL assert_usage(attrs, USAGE_ATTRIBUTES if include_usage else {}) assert any(event["name"] == "First Token Stream Event" for event in span["events"]) @@ -285,7 +279,7 @@ def test_textless_stream_has_no_raw_response_fallback(traced): assert len(seen) == 1 _, attrs = one_span(traced) assert attrs["output.value"] == "" - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL assert_usage(attrs, {}) @@ -304,9 +298,7 @@ def test_synthetic_401_records_error_without_key(traced): assert len(exceptions) == 1 assert "Synthetic fixture rejection" in json.dumps(exceptions) assert LOCAL_KEY not in json.dumps(span) - # traceai-openai records the model only from a non-streamed response; flip - # this when the instrumentor records the request model. - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL assert_usage(attrs, {}) assert_export_separation(traced[0], traced[2]) @@ -688,7 +680,7 @@ def test_app_subprocess_chat_and_stream(tmp_path, stream, env_project): assert attrs["output.value"] == ANSWER assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL if stream: - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL else: assert attrs["gen_ai.request.model"] == RESPONSE_MODEL assert_usage(attrs, {} if stream else USAGE_ATTRIBUTES) @@ -739,26 +731,63 @@ def test_readme_and_requirements_contract(): if ast.unparse(node.func).startswith("os.environ.") and node.args: if isinstance(node.args[0], ast.Constant): assert node.args[0].value in readme - headings = ["## Install", "## Configure", "## Run", "## Code", - "## What you see in Future AGI", "## Provider specifics", "## Privacy", - "## Limits / not covered", "## Tests"] + headings = ["## Install", "## Start the local server", "## Configure", "## Run", + "## Code", "## View traces", "## Privacy", "## Troubleshooting"] assert [readme.index(heading) for heading in headings] == sorted(readme.index(heading) for heading in headings) - for paths in ("python/examples/prismml/src:python:python/frameworks/openai:python/tests", - "python/examples/prismml/src:python/tests"): - assert f'PYTHONPATH="{paths}"' in readme - assert readme.count("pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs") == 2 - assert "--with 'traceAI-openai==0.1.10'" in readme - assert "--with 'fi-instrumentation-otel==1.1.0'" in readme - assert "TBD" not in readme - assert "## Real-server check" in readme - assert "Ternary-Bonsai-2-2B-PQ2_0.gguf" in readme - assert "The automated contract suite uses a local fake" in readme - assert "not 27B tool calling, vision, or MLX" in readme + for diary in ("## Real-server check", "## Tests", "fixture", "Full suite", "future-agi/traceAI#"): + assert diary not in readme + assert "https://api.futureagi.com/tracer/v1/traces" in readme requirements = (ROOT / "requirements.txt").read_text().splitlines() - assert [line for line in requirements if not line.startswith("#")] == [ - "openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0"] + assert [line for line in requirements if line and not line.startswith("#")] == [ + "openai==3.42.1", "traceAI-openai==0.1.12", "fi-instrumentation-otel==1.2.2"] for path in ROOT.rglob("*"): if path.is_file() and "__pycache__" not in path.parts and path.suffix in {".py", ".md", ".txt"}: text = path.read_text() assert not re.search(r"\b[A-Z]{2,5}-\d{3,}\b", text), str(path) assert not re.search(r"\bsk-[A-Za-z0-9]{16,}\b", text), str(path) + + +def test_readme_code_exports_one_span(monkeypatch, capsys): + """Execute the customer example with real registration and OTLP export.""" + readme = (ROOT / "README.md").read_text() + code = re.search(r"## Code\n.*?```python\n(.*?)```", readme, re.S).group(1) + with Receiver() as receiver, FakeOpenAI() as fake: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("PRISMML_BASE_URL", fake.base_url) + monkeypatch.setenv("FI_PROJECT_NAME", "prismml-readme-example") + namespace = {} + try: + exec(compile(code, str(ROOT / "README.md"), "exec"), namespace) + assert capsys.readouterr().out.strip() == ANSWER + assert len(fake.requests()) == 1 + assert fake.requests()[0]["path"] == "/v1/chat/completions" + spans = receiver.spans() + assert len(spans) == 1 + attrs = attributes(spans[0]) + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.request.model"] == RESPONSE_MODEL + assert attrs["output.value"] == ANSWER + assert_usage(attrs, USAGE_ATTRIBUTES) + assert_export_separation(receiver, "prismml-readme-example") + finally: + OpenAIInstrumentor().uninstrument() + if "provider" in namespace: + namespace["provider"].shutdown() + + +def test_ci_covers_published_source_and_sdk_bounds(): + import yaml + + workflow = ROOT.parents[2] / ".github/workflows/prismml-recipe-tests.yml" + job = yaml.safe_load(workflow.read_text())["jobs"]["recipe"] + cells = job["strategy"]["matrix"]["include"] + expected = { + (packages, python, sdk) + for packages in ("published", "source") + for python, sdk in (("3.10", "1.69.0"), ("3.13", "3.42.1")) + } + assert {(cell["packages"], cell["python"], cell["openai"]) for cell in cells} == expected + installs = "\n".join(step.get("run", "") for step in job["steps"]) + assert 'openai==${{ matrix.openai }}' in installs + assert "pip check" in installs From 4d07c0a5e9c381ec59a22b67978d654d6ee2ab34 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 09:48:05 -0800 Subject: [PATCH 2/2] fix(prismml): reject incomplete streams and gate lifecycle release Keep published-package cancellation, early-close and terminal-state contracts strict. Shared instrumentor repairs remain in #520/#536 pending a genuine fixed release. Co-Authored-By: Claude Code --- python/examples/prismml/README.md | 2 +- python/examples/prismml/src/app.py | 17 +- .../tests/test_prismml_cli_lifecycle.py | 134 +++++++ .../prismml/tests/test_prismml_lifecycle.py | 374 ++++++++++++++++++ 4 files changed, 523 insertions(+), 4 deletions(-) create mode 100644 python/examples/prismml/tests/test_prismml_cli_lifecycle.py create mode 100644 python/examples/prismml/tests/test_prismml_lifecycle.py diff --git a/python/examples/prismml/README.md b/python/examples/prismml/README.md index c273054dd..c36405dc4 100644 --- a/python/examples/prismml/README.md +++ b/python/examples/prismml/README.md @@ -61,7 +61,7 @@ python src/app.py --prompt "What is the capital of France?" python src/app.py --stream --prompt "What is the capital of France?" ``` -The application prints the answer and flushes traces before exiting. `--model` overrides `PRISMML_MODEL`. An unset or empty project name uses `prismml-bonsai`. +The application prints the answer and flushes traces before exiting. If a stream ends without a finish reason, it exits with status 1; treat any printed text as a partial answer and retry. `--model` overrides `PRISMML_MODEL`. An unset or empty project name uses `prismml-bonsai`. ## Code diff --git a/python/examples/prismml/src/app.py b/python/examples/prismml/src/app.py index f7e98ef46..8614fa588 100644 --- a/python/examples/prismml/src/app.py +++ b/python/examples/prismml/src/app.py @@ -124,10 +124,21 @@ def main(argv=None) -> int: stream=args.stream, ) if args.stream: - for chunk in response: - if chunk.choices: - print(chunk.choices[0].delta.content or "", end="", flush=True) + completed = False + with response: + for chunk in response: + for choice in chunk.choices: + if choice.index == 0: + print(choice.delta.content or "", end="", flush=True) + if choice.finish_reason is not None: + completed = True print() + # The SDK accepts EOF or [DONE] without a terminal choice. Tracing + # is passive, so the application must reject an incomplete reply. + if not completed: + print("The response ended before a finish reason; retry the request.", + file=sys.stderr) + return 1 else: print(response.choices[0].message.content or "") finally: diff --git a/python/examples/prismml/tests/test_prismml_cli_lifecycle.py b/python/examples/prismml/tests/test_prismml_cli_lifecycle.py new file mode 100644 index 000000000..aaf03c0be --- /dev/null +++ b/python/examples/prismml/tests/test_prismml_cli_lifecycle.py @@ -0,0 +1,134 @@ +"""Execute the unmodified PrismML CLI with real loopback HTTP, OTLP, and SIGINT.""" + +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import pytest + +ROOT = Path(__file__).resolve().parents[1] +from harness import Receiver + + +@pytest.mark.parametrize( + "mode", + ["request_interrupt", "stream_interrupt", "truncated_stream", "premature_done"], +) +def test_real_cli(mode): + request_started = threading.Event() + release = threading.Event() + prefix = { + "id": "synthetic-bonsai", + "object": "chat.completion.chunk", + "created": 1, + "model": "bonsai", + "choices": [ + { + "index": 0, + "delta": {"content": "Partial response"}, + "finish_reason": None, + } + ], + } + frame = ("data: " + json.dumps(prefix) + "\n\n").encode() + + if mode == "premature_done": + frame += b"data: [DONE]\n\n" + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + request_started.set() + if mode == "request_interrupt": + release.wait(15) + return + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + if mode in ("truncated_stream", "premature_done"): + self.send_header("Content-Length", str(len(frame))) + self.end_headers() + self.wfile.write(frame) + self.wfile.flush() + if mode == "stream_interrupt": + release.wait(15) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + proc = None + try: + with Receiver() as receiver: + env = { + k: v + for k, v in os.environ.items() + if not k.startswith(("FI_", "OTEL_", "OPENAI_", "PRISMML_")) + } + env.update( + FI_API_KEY="synthetic-fi-key", + FI_SECRET_KEY="synthetic-fi-secret", + FI_BASE_URL=receiver.origin, + PRISMML_BASE_URL=f"http://127.0.0.1:{server.server_port}/v1", + NO_PROXY="*", + PYTHONDONTWRITEBYTECODE="1", + ) + argv = [ + sys.executable, + str(ROOT / "src/app.py"), + "--prompt", + "safe synthetic input", + ] + if mode != "request_interrupt": + argv.append("--stream") + proc = subprocess.Popen( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env + ) + assert request_started.wait(10), "CLI never reached the local model server" + first = "" + if mode == "stream_interrupt": + first_char = [] + ready = threading.Event() + + def consume_one(): + first_char.append(proc.stdout.read(1)) + ready.set() + + reader = threading.Thread(target=consume_one, daemon=True) + reader.start() + assert ready.wait(10), "CLI did not yield the first streamed token" + reader.join(1) + first = first_char[0] + if mode in ("request_interrupt", "stream_interrupt"): + proc.send_signal(signal.SIGINT) + out, err = proc.communicate(timeout=10) + assert proc.returncode != 0, "Incomplete or interrupted CLI must fail" + spans = receiver.spans() + print( + json.dumps( + { + "scenario": mode, + "exit_code": proc.returncode, + "stdout": first + out, + "span_count": len(spans), + "statuses": [s.get("status", {}) for s in spans], + } + ) + ) + assert len(spans) == 1, f"{mode}: actual CLI exported {len(spans)} spans" + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR", ( + f"{mode}: actual CLI returned {proc.returncode} and reported {spans[0]['status']}" + ) + finally: + release.set() + if proc and proc.poll() is None: + proc.kill() + proc.communicate() + server.shutdown() + server.server_close() + thread.join(2) diff --git a/python/examples/prismml/tests/test_prismml_lifecycle.py b/python/examples/prismml/tests/test_prismml_lifecycle.py new file mode 100644 index 000000000..0ee96acf7 --- /dev/null +++ b/python/examples/prismml/tests/test_prismml_lifecycle.py @@ -0,0 +1,374 @@ +"""Release gates for maintained-instrumentor lifecycle behavior. + +These contracts must pass with the recipe pins, not only an unreleased source tree. +Shared repairs live in traceAI #520/#536; keep this route blocked until released. +""" + +import asyncio +import json +import os + +import app +from _fake_openai import chat_response, stream_body +from harness import Receiver +import httpx +import openai +import pytest +from traceai_openai import OpenAIInstrumentor + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + for name in list(os.environ): + if name.startswith(("FI_", "OTEL_", "OPENAI_", "PRISMML_")) or name.lower() in ( + "http_proxy", + "https_proxy", + "all_proxy", + "no_proxy", + ): + monkeypatch.delenv(name) + monkeypatch.setenv("FI_API_KEY", "synthetic-review-fi-key") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-review-fi-secret") + monkeypatch.setenv("NO_PROXY", "*") + yield + if OpenAIInstrumentor().is_instrumented_by_opentelemetry: + OpenAIInstrumentor().uninstrument() + + +@pytest.fixture +def traced(monkeypatch): + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing("prismml-independent-review") + try: + yield receiver, provider + finally: + OpenAIInstrumentor().uninstrument() + provider.shutdown() + + +def client_factory(handler): + return lambda **kwargs: openai.OpenAI( + **kwargs, + max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + + +def attrs(span): + return { + item["key"]: next(iter(item["value"].values())) + for item in span.get("attributes", []) + } + + +def first_chunk(): + return stream_body().split(b"\n\n", 1)[0] + b"\n\n" + + +@pytest.mark.parametrize("stream", [False, True]) +def test_exact_app_interrupt_exports_error(monkeypatch, traced, stream): + receiver, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + + class Interrupted(httpx.SyncByteStream): + def __iter__(self): + yield first_chunk() + raise KeyboardInterrupt("synthetic interrupt") + + def handler(request): + if not stream: + raise KeyboardInterrupt("synthetic interrupt") + return httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, stream=Interrupted() + ) + + monkeypatch.setattr(app, "OpenAI", client_factory(handler)) + with pytest.raises(KeyboardInterrupt): + app.main(["--stream"] if stream else []) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1, ( + f"Interrupted app exported {len(spans)} spans after its finally flush" + ) + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +@pytest.mark.parametrize("terminator", [b"", b"data: [DONE]\n\n"]) +def test_exact_app_incomplete_stream_is_not_success(monkeypatch, traced, terminator): + receiver, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + + def handler(request): + return httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + content=first_chunk() + terminator, + ) + + monkeypatch.setattr(app, "OpenAI", client_factory(handler)) + outcome = app.main(["--stream"]) + assert outcome == 1 + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + status = spans[0].get("status", {}).get("code") + assert status == "STATUS_CODE_ERROR", ( + f"Incomplete reply: app returned {outcome}, span status={status}, output={attrs(spans[0]).get('output.value')!r}" + ) + + +def test_explicit_stream_close_finishes_span(traced): + receiver, provider = traced + with client_factory( + lambda req: httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, content=stream_body() + ) + )(base_url="http://127.0.0.1:9/v1", api_key="synthetic-model-key") as client: + stream = client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe prompt"}], + stream=True, + ) + next(stream) + stream.close() + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1, f"Explicitly closed stream exported {len(spans)} spans" + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_async_request_cancellation_finishes_span(traced): + receiver, provider = traced + + async def run(): + async def handler(request): + raise asyncio.CancelledError("synthetic cancellation") + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-model-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + with pytest.raises(asyncio.CancelledError): + await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe prompt"}], + ) + + asyncio.run(run()) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1, f"Cancelled async request exported {len(spans)} spans" + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_parent_context_and_double_instrumentation(traced): + receiver, provider = traced + OpenAIInstrumentor().instrument(tracer_provider=provider) + tracer = provider.get_tracer("synthetic-parent") + with tracer.start_as_current_span("parent"): + with client_factory(lambda req: httpx.Response(200, json=chat_response()))( + base_url="http://127.0.0.1:9/v1", api_key="synthetic-model-key" + ) as client: + client.chat.completions.create( + model="bonsai", messages=[{"role": "user", "content": "safe prompt"}] + ) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 2 + parent = next(s for s in spans if s["name"] == "parent") + child = next(s for s in spans if s["name"] == "ChatCompletion") + assert child["parentSpanId"] == parent["spanId"] + assert child["traceId"] == parent["traceId"] + assert attrs(child)["gen_ai.span.kind"] == "LLM" + + +@pytest.mark.parametrize("stream", [False, True]) +def test_async_success_has_one_span(traced, stream): + receiver, provider = traced + + async def run(): + def handler(request): + return ( + httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + content=stream_body(), + ) + if stream + else httpx.Response(200, json=chat_response()) + ) + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-model-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + response = await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe prompt"}], + stream=stream, + ) + if stream: + async for _ in response: + pass + + asyncio.run(run()) + provider.force_flush() + assert len(receiver.spans()) == 1 + assert receiver.spans()[0]["status"]["code"] == "STATUS_CODE_OK" + + +def test_hidden_errors_do_not_export_content(monkeypatch): + for key in ( + "FI_HIDE_INPUTS", + "FI_HIDE_OUTPUTS", + "FI_HIDE_LLM_INVOCATION_PARAMETERS", + ): + monkeypatch.setenv(key, "true") + marker = "synthetic-private-prompt-and-answer" + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing("prismml-error-privacy") + try: + with client_factory( + lambda req: httpx.Response(400, json={"error": {"message": marker}}) + )( + base_url="http://127.0.0.1:9/v1", api_key="synthetic-model-key" + ) as client: + with pytest.raises(openai.BadRequestError): + client.chat.completions.create( + model="bonsai", messages=[{"role": "user", "content": marker}] + ) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + assert marker not in json.dumps(spans) + finally: + OpenAIInstrumentor().uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize( + "body", + [b"", first_chunk(), first_chunk() + b"data: [DONE]\n\n", b"data: [DONE]\n\n"], +) +def test_cli_rejects_missing_terminal_choice(monkeypatch, traced, capsys, body): + _, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + response = httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, content=body + ) + monkeypatch.setattr(app, "OpenAI", client_factory(lambda request: response)) + assert app.main(["--stream"]) == 1 + assert "before a finish reason" in capsys.readouterr().err + assert response.is_closed + + +@pytest.mark.parametrize("reason", ["stop", "length", "tool_calls", "content_filter"]) +@pytest.mark.parametrize("usage_tail", [False, True]) +def test_cli_accepts_terminal_choice_and_usage_tail( + monkeypatch, traced, reason, usage_tail +): + _, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + terminal = { + "id": "synthetic-terminal", + "object": "chat.completion.chunk", + "created": 1, + "model": "bonsai", + "choices": [{"index": 0, "delta": {}, "finish_reason": reason}], + } + body = first_chunk() + ("data: " + json.dumps(terminal) + "\n\n").encode() + if usage_tail: + usage = dict( + terminal, + choices=[], + usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + ) + body += ("data: " + json.dumps(usage) + "\n\n").encode() + body += b"data: [DONE]\n\n" + response = httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, content=body + ) + monkeypatch.setattr(app, "OpenAI", client_factory(lambda request: response)) + assert app.main(["--stream"]) == 0 + assert response.is_closed + + +def test_async_stream_cancellation_finishes_span(traced): + receiver, provider = traced + interruption = asyncio.CancelledError("synthetic stream cancellation") + + class Interrupted(httpx.AsyncByteStream): + async def __aiter__(self): + yield first_chunk() + raise interruption + + async def run(): + async def handler(request): + return httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, stream=Interrupted() + ) + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + stream = await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe"}], + stream=True, + ) + try: + with pytest.raises(asyncio.CancelledError) as caught: + async for _ in stream: + pass + assert caught.value is interruption + finally: + await stream.close() + + asyncio.run(run()) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_async_explicit_close_finishes_span_once(traced): + receiver, provider = traced + + async def run(): + async def handler(request): + return httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + content=stream_body(), + ) + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + stream = await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe"}], + stream=True, + ) + await stream.__anext__() + await stream.close() + await stream.close() + + asyncio.run(run()) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR"