diff --git a/.github/workflows/prismml-recipe-tests.yml b/.github/workflows/prismml-recipe-tests.yml index a7ca8deaf..d06063e76 100644 --- a/.github/workflows/prismml-recipe-tests.yml +++ b/.github/workflows/prismml-recipe-tests.yml @@ -16,13 +16,25 @@ permissions: jobs: recipe: - name: PrismML recipe (${{ matrix.packages }}) + name: PrismML recipe (${{ matrix.packages }}, Python ${{ matrix.python }}, OpenAI ${{ matrix.openai }}) runs-on: ubuntu-latest timeout-minutes: 10 strategy: fail-fast: false matrix: - packages: [published, source] + include: + - packages: published + python: '3.10' + openai: '1.69.0' + - packages: source + python: '3.10' + openai: '1.69.0' + - packages: published + python: '3.13' + openai: '3.42.1' + - packages: source + python: '3.13' + openai: '3.42.1' env: FI_BASE_URL: http://127.0.0.1:9 FI_GRPC_URL: http://127.0.0.1:9 @@ -35,11 +47,15 @@ jobs: persist-credentials: false - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: ${{ matrix.python }} - name: Install published recipe pins and test tools - run: pip install -r python/examples/prismml/requirements.txt pytest httpx 'wrapt<2' + run: pip install -r python/examples/prismml/requirements.txt pytest httpx 'wrapt<2' PyYAML + - name: Select SDK compatibility version + run: pip install 'openai==${{ matrix.openai }}' - name: Install source distributions if: matrix.packages == 'source' run: pip install ./python ./python/frameworks/openai + - name: Check dependency consistency + run: pip check - name: Loopback-only recipe contracts run: pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs diff --git a/python/examples/prismml/README.md b/python/examples/prismml/README.md index 6a9470ada..c36405dc4 100644 --- a/python/examples/prismml/README.md +++ b/python/examples/prismml/README.md @@ -1,203 +1,131 @@ # PrismML (OpenAI-compatible) with traceAI -PrismML here means the Bonsai local server from prismml.com; this recipe traces Chat Completions through the `openai` SDK at `http://localhost:8080/v1` ([server docs](https://docs.prismml.com/run/server)). It uses `traceai-openai`, with no PrismML instrumentation package. - -Not these: other products named Prism are not covered here, for example the [ssimplifi Prism proxy guide](https://ssimplifi.com/guides/openai-compatible-api), [prism-proxy](https://git.sovereign-society.org/prism/prism-proxy) and [Prism PHP](https://mintlify.wiki/prism-php/prism/providers/overview). +Trace calls to your local [PrismML Bonsai server](https://docs.prismml.com/run/server) in Future AGI using the maintained `traceai-openai` package. This recipe uses the OpenAI Python SDK's Chat Completions API; it is not an integration for other products named Prism. ## Install -From this recipe directory, install the pinned dependencies: +Use Python 3.10–3.13. From this recipe directory: + +```bash +python -m pip install -r requirements.txt +``` + +For an existing application, install the same packages directly: ```bash -pip install -r requirements.txt +python -m pip install 'openai==3.42.1' 'traceAI-openai==0.1.12' 'fi-instrumentation-otel==1.2.2' ``` -The plain installation form is: +## Start the local server + +Follow the [PrismML server setup](https://docs.prismml.com/run/server) to download a model and start the llama.cpp server. In the Bonsai demo directory, give the loaded model a stable name: ```bash -pip install traceAI-openai fi-instrumentation-otel openai +./scripts/start_llama_server.sh --alias bonsai ``` -The test versions and verification scope are listed below. +The API base URL is `http://localhost:8080/v1`. Keep the server on loopback: it does not authenticate requests. Binding to a non-loopback address exposes it to your network; use a trusted network or firewall the port. + +On Apple Silicon, PrismML also provides `./scripts/start_mlx_server.sh` at `http://localhost:8081/v1`. Follow its [sampling and thinking configuration](https://docs.prismml.com/run/server); unlike the llama.cpp launcher, the MLX server expects those settings in each request. ## Configure -Start your server first, following the [PrismML server docs](https://docs.prismml.com/run/server): `./scripts/start_llama_server.sh` serves llama.cpp on port 8080; `./scripts/start_mlx_server.sh` serves MLX on Apple Silicon on port 8081. Both expose the same OpenAI-compatible API. +Set your Future AGI credentials in the shell that runs the Python client: -| Variable | Value or default | Destination | -|---|---|---| -| `FI_API_KEY` | Your Future AGI API key | Tracer only; never the local server | -| `FI_SECRET_KEY` | Your Future AGI secret key | Tracer only; never the local server | -| `PRISMML_API_KEY` | Optional; `not-needed` | OpenAI client only; never the Future AGI export | -| `PRISMML_MODEL` | Optional; `bonsai` | Model name sent by the OpenAI client | -| `PRISMML_BASE_URL` | Optional; `http://localhost:8080/v1` | OpenAI client; use `http://localhost:8081/v1` for MLX | -| `FI_BASE_URL` | Optional; the tracer's Future AGI collector default | Tracer export origin | -| `FI_PROJECT_NAME` | Optional; `prismml-bonsai` | Tracer only; the Observe project name | -| `FI_HIDE_INPUTS` | Optional; `true` masks input attributes | Tracer only | -| `FI_HIDE_OUTPUTS` | Optional; `true` masks output attributes | Tracer only | -| `FI_HIDE_LLM_INVOCATION_PARAMETERS` | Optional; `true` omits invocation parameters | Tracer only | +```bash +export FI_API_KEY="YOUR_FUTURE_AGI_API_KEY" +export FI_SECRET_KEY="YOUR_FUTURE_AGI_SECRET_KEY" +export FI_PROJECT_NAME="prismml-bonsai" +export PRISMML_BASE_URL="http://localhost:8080/v1" +export PRISMML_MODEL="bonsai" +``` -The local key is a placeholder, not a Future AGI secret. The OpenAI SDK requires a non-empty key, but the server does not check it ([server docs](https://docs.prismml.com/run/server)). An unset or empty `PRISMML_API_KEY` uses `not-needed`; an unset or empty `PRISMML_MODEL` uses `bonsai`. +The tracer sends OTLP to `https://api.futureagi.com/tracer/v1/traces` by default. The OpenAI client sends model requests only to `PRISMML_BASE_URL`. Do not use your Future AGI credentials as the local server key. -The recipe validates `PRISMML_BASE_URL` before tracing. Invalid URLs exit 2 and name that variable. An explicitly empty `--model` exits 2 and names `PRISMML_MODEL`. The recipe does not validate the Future AGI keys. +| Variable | Value or default | Purpose | +|---|---|---| +| `FI_API_KEY` | Your Future AGI API key | Authenticate trace export | +| `FI_SECRET_KEY` | Your Future AGI secret key | Authenticate trace export | +| `FI_PROJECT_NAME` | Optional; `prismml-bonsai` | Observe project name | +| `FI_BASE_URL` | Optional; `https://api.futureagi.com` | Trace export origin; omit `/tracer/v1/traces` | +| `PRISMML_BASE_URL` | Optional; `http://localhost:8080/v1` | Local server API base URL | +| `PRISMML_MODEL` | Optional; `bonsai` | Requested model name | +| `PRISMML_API_KEY` | Optional; `not-needed` | Non-empty placeholder required by the OpenAI SDK | ## Run -From this recipe directory, with your server already running: +Return to this recipe directory and run a normal or streamed completion: ```bash -export FI_API_KEY="placeholder-fi-api-key" -export FI_SECRET_KEY="placeholder-fi-secret-key" -export PRISMML_API_KEY="not-needed" -export PRISMML_MODEL="bonsai" -export PRISMML_BASE_URL="http://localhost:8080/v1" -python src/app.py -python src/app.py --stream +python src/app.py --prompt "What is the capital of France?" +python src/app.py --stream --prompt "What is the capital of France?" ``` -Replace the Future AGI placeholders with your keys. `--prompt` changes the question and `--model` overrides `PRISMML_MODEL`. Set `FI_PROJECT_NAME` to send the traces to another Observe project; unset or empty, they go to `prismml-bonsai`. The app prints answer text and flushes the tracer. +The application prints the answer and flushes traces before exiting. If a stream ends without a finish reason, it exits with status 1; treat any printed text as a partial answer and retry. `--model` overrides `PRISMML_MODEL`. An unset or empty project name uses `prismml-bonsai`. ## Code -Instrument before creating the client. Future AGI keys are read by the tracer from its environment. +For your own application, register the tracer and instrument the SDK once, before creating the client. Do not add a second OpenAI instrumentor around the same requests. ```python import os + from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType from openai import OpenAI from traceai_openai import OpenAIInstrumentor -provider = register( # FI_PROJECT_NAME overrides, as in src/app.py - project_name=os.environ.get("FI_PROJECT_NAME") or "prismml-bonsai", project_type=ProjectType.OBSERVE, - set_global_tracer_provider=False, verbose=False, + +provider = register( + project_name=os.environ.get("FI_PROJECT_NAME") or "prismml-bonsai", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, ) OpenAIInstrumentor().instrument(tracer_provider=provider) -with OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed") as client: - response = client.chat.completions.create( - model="bonsai", messages=[{"role": "user", "content": "Say hello."}], stream=False, - ) - print(response.choices[0].message.content or "") -provider.force_flush() +try: + with OpenAI( + base_url=os.environ.get("PRISMML_BASE_URL") or "http://localhost:8080/v1", + api_key=os.environ.get("PRISMML_API_KEY") or "not-needed", + ) as client: + response = client.chat.completions.create( + model=os.environ.get("PRISMML_MODEL") or "bonsai", + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + print(response.choices[0].message.content or "") +finally: + provider.force_flush() ``` -## What you see in Future AGI - -The fixtures produce one `ChatCompletion` span per call with `gen_ai.span.kind=LLM`. The provider field says `openai`: `gen_ai.provider.name` is the shared instrumentor's label. - -Current `traceai-openai` behavior: - -- For a successful non-streamed call, `gen_ai.request.model` is the model name the server returns. The fixture requests `bonsai` and returns the synthetic loaded-model name `Ternary-Bonsai-2-27B-Q4_K_M.gguf`; the latter is recorded. The requested `bonsai` remains in the `gen_ai.request.parameters` JSON. A real `llama-server` started by the demo scripts returns the loaded file's absolute path, such as `/Users/you/Bonsai-demo/models/gguf/1.7B/Bonsai-1.7B-Q1_0.gguf`, and that path is what gets recorded. Start the server with `./scripts/start_llama_server.sh --alias bonsai` to record `bonsai` instead; the script passes extra arguments to `llama-server`. -- Up to `traceai-openai` 0.1.10, streamed and failed calls omit `gen_ai.request.model`; later versions (future-agi/traceAI#249) record the requested model there. The requested model remains in `gen_ai.request.parameters`, unless invocation parameters are hidden. -- Non-streamed token usage is recorded as `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, and `gen_ai.usage.total_tokens` when the response supplies it. Missing usage is omitted, never replaced with zero. -- A stream accumulates answer text in `output.value`. Default fixture streams omit usage. With `stream_options={"include_usage": True}` and a final usage chunk, the same three token attributes are recorded. This verifies the instrumentor, not the real server's streaming usage support. -- Non-streamed `output.value` contains assistant content. Tool-call responses contain a readable function call instead. A stream with no text records `output.value=''`; Chat Completions have no raw-response fallback here. -- SDK errors set span status to ERROR and record an exception event. The 401 fixture is synthetic; it does not imply that this unauthenticated server checks a key. - -A cost lookup keyed on the requested name may miss the server's returned name. Cost may be unknown when usage or a matching price entry is unavailable. This recipe adds no model pricing. - -## Provider specifics - -According to the [PrismML server docs](https://docs.prismml.com/run/server): - -| Server | API base URL | -|---|---| -| llama.cpp | `http://localhost:8080/v1` | -| MLX on Apple Silicon | `http://localhost:8081/v1` | +## View traces -The SDK appends a trailing slash to its stored base URL and calls `/v1/chat/completions`. The validator returns allowed URLs unchanged. It permits exactly `/v1` or `/v1/` on any host and refuses the root web chat, a full `/v1/chat/completions` URL, other paths, credentials, query or fragment delimiters, whitespace, control characters, non-ASCII hosts and malformed IDNA. Host names may use letters, digits, hyphens and underscores inside a label, so container or Compose service names such as `bonsai_server` work. +Open **Observe** in Future AGI and select the project named by `FI_PROJECT_NAME`. Each completion produces one `ChatCompletion` LLM span with the prompt, answer, latency and any token usage supplied by the server. The provider field says `openai` because this local endpoint uses the shared OpenAI instrumentor. -Loopback HTTP is allowed without a warning for `localhost` and loopback IP addresses (`127.0.0.0/8`, `::1`). HTTP outside loopback is allowed with one stderr warning about the unauthenticated server. Use it only on a trusted LAN. HTTPS URLs still need the same API path. +- Non-streamed calls record the server's returned model in `gen_ai.request.model`. Streamed and failed calls record the requested model. The original requested name also appears in `gen_ai.request.parameters` unless invocation parameters are hidden. +- Streamed answer text is accumulated in `output.value`. To request streaming token counts, pass `stream_options={"include_usage": True}`; counts appear only if the server returns a final usage chunk. Missing usage is omitted, not replaced with zero. +- SDK errors mark the span as ERROR. Returned `tool_calls` are recorded, but the instrumentor does not execute tools. +- Local models may have no matching price entry, so cost can be unavailable even when token counts are present. -The [server docs](https://docs.prismml.com/run/server) warn: - -> The scripts bind to `127.0.0.1`, so the server is reachable only from the same machine and there is no authentication. Setting `BONSAI_HOST` to any non-loopback address (for example `0.0.0.0`) exposes an unauthenticated server to your network. - -For llama-server, any model string works because it serves the loaded model; `GET /v1/models` returns the real name: the loaded file's absolute path by default, or the name set with `--alias`. The documented Python example requests `bonsai` with streaming ([server docs](https://docs.prismml.com/run/server)). `traceai-openai` records the `model` field of the server's non-streamed response, pinned by the tests; the loaded-model name used in tests is synthetic, not a promised filename. - -Thinking is controlled per request with `thinking_budget_tokens`; zero disables it ([server docs](https://docs.prismml.com/run/server)). Send it through the SDK's `extra_body`: - -```python -response = client.chat.completions.create( - model="bonsai", messages=[{"role": "user", "content": "Say hello."}], - extra_body={"thinking_budget_tokens": 0}, -) -``` - -The tests pin that this value reaches the HTTP request but is absent from `gen_ai.request.parameters`. The SDK merges `extra_body` after the instrumentor reads the request parameters. - -The 27B model supports native OpenAI-style `tool_calls` ([server docs](https://docs.prismml.com/run/server)). The fixture uses a standard `tools` array. It records definitions in `gen_ai.tool.definitions` and `gen_ai.tool.definitions.0.tool.json_schema`, and records the returned tool ID, function name and arguments under `gen_ai.output.messages.0.message.tool_calls.0.tool_call.*`. It does not execute a tool. +For llama.cpp thinking control, pass `extra_body={"thinking_budget_tokens": 0}` to `client.chat.completions.create()` to disable thinking for that request. This server-specific option reaches the model server but is not included in the pinned instrumentor's invocation parameters. See the [PrismML server reference](https://docs.prismml.com/run/server) for model-specific settings. ## Privacy -Prompt and answer text normally leave the machine in the Future AGI trace export. Set `FI_HIDE_INPUTS=true` to mask input attributes and `FI_HIDE_OUTPUTS=true` to mask output attributes. The tests verify each flag against visible controls for both chat and streaming. - -The server's returned model name is exported too, and these flags do not mask it. Without `--alias`, `llama-server` returns the model file's absolute path, which can include your user name or directory layout. Start the server with `--alias bonsai` to keep local paths out of the export. - -`FI_HIDE_INPUTS` masks only inputs. If the server echoes the prompt in its answer, that text remains in output attributes unless outputs are also hidden. Non-streamed output is assistant content, and streamed output is accumulated text, including an empty string when no text arrives. This recipe does not export a raw Chat Completions response in `output.value`. - -The local server still receives the prompt. Future AGI masking does not change the server's memory or its own logging. Key separation tests verify that `not-needed` and the custom local placeholder never reach the Future AGI export, while Future AGI keys never reach the local request. - -## Limits / not covered - -- The built-in web chat, Open WebUI, MCP, Responses API, embeddings and other API surfaces. -- Starting or stopping the server, distributing weights, live GPU jobs, MLX server differences, benchmarks or model quality. -- Tool execution and streaming tool calls. -- Other products called Prism: the [ssimplifi Prism proxy guide](https://ssimplifi.com/guides/openai-compatible-api), [prism-proxy](https://git.sovereign-society.org/prism/prism-proxy), and [Prism PHP](https://mintlify.wiki/prism-php/prism/providers/overview). -- The automated contract suite uses a local fake of the OpenAI API and does not download weights. The real-server check below is separate. - -## Real-server check - -On 2026-10-28 this recipe ran against the official PrismML `llama-server` -release `prism-b10826-c9c34bc`, with the official -`Ternary-Bonsai-2-2B-PQ2_0.gguf` weights (verified against the published SHA256SUMS). -The Linux CPU server used `BONSAI_MODEL=2B`, `BONSAI_NGL=0`, `BONSAI_CTX=4096`, -`BONSAI_THINKING=0`, and `--alias bonsai`. The model and binary came from the -[official demo](https://github.com/PrismML-Eng/Bonsai-demo) and its linked releases. - -With the pinned `openai==3.24.0`, `traceAI-openai==0.1.10`, and -`fi-instrumentation-otel==1.1.0`, both the normal and `--stream` recipe commands -returned answers. Their exported spans were read back in Future AGI Cloud: -non-streamed calls had model `bonsai` and token usage; the default stream had -answer text but no model or usage attributes, matching the published-version -behavior described above. A separate call with input/output/invocation-parameter -hiding enabled exported no prompt or answer text. This check covers the 2B text -model on the llama.cpp server, not 27B tool calling, vision, or MLX. - -## Tests - -Run these commands from the repository root. All provider responses are synthetic: in-process MockTransport at both documented URLs, plus a server and OTLP receiver bound to `127.0.0.1`. A subprocess socket guard refuses non-loopback connections before DNS and has a loopback positive control. - -Repository source: +Model execution is local, but prompt and answer content are exported to Future AGI by default. To hide content and invocation parameters, set these variables before running the application: ```bash -env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ - PYTHONPATH="python/examples/prismml/src:python:python/frameworks/openai:python/tests" \ - uv run --no-project --python 3.11 \ - --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ - --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ - --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ - --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ - pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +export FI_HIDE_INPUTS=true +export FI_HIDE_OUTPUTS=true +export FI_HIDE_LLM_INVOCATION_PARAMETERS=true ``` -Published packages: +Hide both input and output if an answer might echo a sensitive prompt. These settings do not change what the local server receives or logs. -```bash -env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ - PYTHONPATH="python/examples/prismml/src:python/tests" \ - uv run --no-project --python 3.11 \ - --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ - --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ - --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ - --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ - --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ - pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs -``` +The returned model name is metadata and is not masked by these flags. Without an alias, llama-server can return the loaded model file's **absolute path**, including your user name or directory layout. Start it with `--alias bonsai` to keep local paths out of traces. + +## Troubleshooting -To check the `traceai-openai` SDK floor, change the source command's OpenAI pin to `openai==1.69.0`. +- **Connection refused:** start the local server first and check the port. Use port 8080 for llama.cpp or 8081 for MLX. +- **URL validation error:** the recipe requires a base URL ending in `/v1` or `/v1/`, not the web chat root or `/v1/chat/completions`. Remove credentials, query strings and fragments. Compose host names such as `bonsai_server` are accepted; non-loopback HTTP prints a security warning. +- **No traces:** check `FI_API_KEY`, `FI_SECRET_KEY` and the project name. `FI_BASE_URL` is the trace export origin, not the model server URL. +- **Unexpected model path or missing cost:** use `--alias bonsai`; pricing still requires a matching model price entry. -| Python | `openai` | traceAI packages | Result | -|---|---|---|---| -| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | -| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | -| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | +This route traces calls made by your Python client. It does not automatically trace the server's built-in web chat or other applications connected to it. diff --git a/python/examples/prismml/requirements.txt b/python/examples/prismml/requirements.txt index 2e789e40c..5ede74902 100644 --- a/python/examples/prismml/requirements.txt +++ b/python/examples/prismml/requirements.txt @@ -1,4 +1,4 @@ -# Installation pins; see README for source verification and the SDK floor. -openai==3.24.0 -traceAI-openai==0.1.10 -fi-instrumentation-otel==1.1.0 +# Published dependencies for the local Bonsai server recipe. +openai==3.42.1 +traceAI-openai==0.1.12 +fi-instrumentation-otel==1.2.2 diff --git a/python/examples/prismml/src/app.py b/python/examples/prismml/src/app.py index f7e98ef46..8614fa588 100644 --- a/python/examples/prismml/src/app.py +++ b/python/examples/prismml/src/app.py @@ -124,10 +124,21 @@ def main(argv=None) -> int: stream=args.stream, ) if args.stream: - for chunk in response: - if chunk.choices: - print(chunk.choices[0].delta.content or "", end="", flush=True) + completed = False + with response: + for chunk in response: + for choice in chunk.choices: + if choice.index == 0: + print(choice.delta.content or "", end="", flush=True) + if choice.finish_reason is not None: + completed = True print() + # The SDK accepts EOF or [DONE] without a terminal choice. Tracing + # is passive, so the application must reject an incomplete reply. + if not completed: + print("The response ended before a finish reason; retry the request.", + file=sys.stderr) + return 1 else: print(response.choices[0].message.content or "") finally: diff --git a/python/examples/prismml/tests/test_prismml_cli_lifecycle.py b/python/examples/prismml/tests/test_prismml_cli_lifecycle.py new file mode 100644 index 000000000..aaf03c0be --- /dev/null +++ b/python/examples/prismml/tests/test_prismml_cli_lifecycle.py @@ -0,0 +1,134 @@ +"""Execute the unmodified PrismML CLI with real loopback HTTP, OTLP, and SIGINT.""" + +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import pytest + +ROOT = Path(__file__).resolve().parents[1] +from harness import Receiver + + +@pytest.mark.parametrize( + "mode", + ["request_interrupt", "stream_interrupt", "truncated_stream", "premature_done"], +) +def test_real_cli(mode): + request_started = threading.Event() + release = threading.Event() + prefix = { + "id": "synthetic-bonsai", + "object": "chat.completion.chunk", + "created": 1, + "model": "bonsai", + "choices": [ + { + "index": 0, + "delta": {"content": "Partial response"}, + "finish_reason": None, + } + ], + } + frame = ("data: " + json.dumps(prefix) + "\n\n").encode() + + if mode == "premature_done": + frame += b"data: [DONE]\n\n" + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + request_started.set() + if mode == "request_interrupt": + release.wait(15) + return + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + if mode in ("truncated_stream", "premature_done"): + self.send_header("Content-Length", str(len(frame))) + self.end_headers() + self.wfile.write(frame) + self.wfile.flush() + if mode == "stream_interrupt": + release.wait(15) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + proc = None + try: + with Receiver() as receiver: + env = { + k: v + for k, v in os.environ.items() + if not k.startswith(("FI_", "OTEL_", "OPENAI_", "PRISMML_")) + } + env.update( + FI_API_KEY="synthetic-fi-key", + FI_SECRET_KEY="synthetic-fi-secret", + FI_BASE_URL=receiver.origin, + PRISMML_BASE_URL=f"http://127.0.0.1:{server.server_port}/v1", + NO_PROXY="*", + PYTHONDONTWRITEBYTECODE="1", + ) + argv = [ + sys.executable, + str(ROOT / "src/app.py"), + "--prompt", + "safe synthetic input", + ] + if mode != "request_interrupt": + argv.append("--stream") + proc = subprocess.Popen( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env + ) + assert request_started.wait(10), "CLI never reached the local model server" + first = "" + if mode == "stream_interrupt": + first_char = [] + ready = threading.Event() + + def consume_one(): + first_char.append(proc.stdout.read(1)) + ready.set() + + reader = threading.Thread(target=consume_one, daemon=True) + reader.start() + assert ready.wait(10), "CLI did not yield the first streamed token" + reader.join(1) + first = first_char[0] + if mode in ("request_interrupt", "stream_interrupt"): + proc.send_signal(signal.SIGINT) + out, err = proc.communicate(timeout=10) + assert proc.returncode != 0, "Incomplete or interrupted CLI must fail" + spans = receiver.spans() + print( + json.dumps( + { + "scenario": mode, + "exit_code": proc.returncode, + "stdout": first + out, + "span_count": len(spans), + "statuses": [s.get("status", {}) for s in spans], + } + ) + ) + assert len(spans) == 1, f"{mode}: actual CLI exported {len(spans)} spans" + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR", ( + f"{mode}: actual CLI returned {proc.returncode} and reported {spans[0]['status']}" + ) + finally: + release.set() + if proc and proc.poll() is None: + proc.kill() + proc.communicate() + server.shutdown() + server.server_close() + thread.join(2) diff --git a/python/examples/prismml/tests/test_prismml_lifecycle.py b/python/examples/prismml/tests/test_prismml_lifecycle.py new file mode 100644 index 000000000..0ee96acf7 --- /dev/null +++ b/python/examples/prismml/tests/test_prismml_lifecycle.py @@ -0,0 +1,374 @@ +"""Release gates for maintained-instrumentor lifecycle behavior. + +These contracts must pass with the recipe pins, not only an unreleased source tree. +Shared repairs live in traceAI #520/#536; keep this route blocked until released. +""" + +import asyncio +import json +import os + +import app +from _fake_openai import chat_response, stream_body +from harness import Receiver +import httpx +import openai +import pytest +from traceai_openai import OpenAIInstrumentor + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + for name in list(os.environ): + if name.startswith(("FI_", "OTEL_", "OPENAI_", "PRISMML_")) or name.lower() in ( + "http_proxy", + "https_proxy", + "all_proxy", + "no_proxy", + ): + monkeypatch.delenv(name) + monkeypatch.setenv("FI_API_KEY", "synthetic-review-fi-key") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-review-fi-secret") + monkeypatch.setenv("NO_PROXY", "*") + yield + if OpenAIInstrumentor().is_instrumented_by_opentelemetry: + OpenAIInstrumentor().uninstrument() + + +@pytest.fixture +def traced(monkeypatch): + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing("prismml-independent-review") + try: + yield receiver, provider + finally: + OpenAIInstrumentor().uninstrument() + provider.shutdown() + + +def client_factory(handler): + return lambda **kwargs: openai.OpenAI( + **kwargs, + max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + + +def attrs(span): + return { + item["key"]: next(iter(item["value"].values())) + for item in span.get("attributes", []) + } + + +def first_chunk(): + return stream_body().split(b"\n\n", 1)[0] + b"\n\n" + + +@pytest.mark.parametrize("stream", [False, True]) +def test_exact_app_interrupt_exports_error(monkeypatch, traced, stream): + receiver, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + + class Interrupted(httpx.SyncByteStream): + def __iter__(self): + yield first_chunk() + raise KeyboardInterrupt("synthetic interrupt") + + def handler(request): + if not stream: + raise KeyboardInterrupt("synthetic interrupt") + return httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, stream=Interrupted() + ) + + monkeypatch.setattr(app, "OpenAI", client_factory(handler)) + with pytest.raises(KeyboardInterrupt): + app.main(["--stream"] if stream else []) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1, ( + f"Interrupted app exported {len(spans)} spans after its finally flush" + ) + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +@pytest.mark.parametrize("terminator", [b"", b"data: [DONE]\n\n"]) +def test_exact_app_incomplete_stream_is_not_success(monkeypatch, traced, terminator): + receiver, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + + def handler(request): + return httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + content=first_chunk() + terminator, + ) + + monkeypatch.setattr(app, "OpenAI", client_factory(handler)) + outcome = app.main(["--stream"]) + assert outcome == 1 + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + status = spans[0].get("status", {}).get("code") + assert status == "STATUS_CODE_ERROR", ( + f"Incomplete reply: app returned {outcome}, span status={status}, output={attrs(spans[0]).get('output.value')!r}" + ) + + +def test_explicit_stream_close_finishes_span(traced): + receiver, provider = traced + with client_factory( + lambda req: httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, content=stream_body() + ) + )(base_url="http://127.0.0.1:9/v1", api_key="synthetic-model-key") as client: + stream = client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe prompt"}], + stream=True, + ) + next(stream) + stream.close() + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1, f"Explicitly closed stream exported {len(spans)} spans" + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_async_request_cancellation_finishes_span(traced): + receiver, provider = traced + + async def run(): + async def handler(request): + raise asyncio.CancelledError("synthetic cancellation") + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-model-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + with pytest.raises(asyncio.CancelledError): + await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe prompt"}], + ) + + asyncio.run(run()) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1, f"Cancelled async request exported {len(spans)} spans" + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_parent_context_and_double_instrumentation(traced): + receiver, provider = traced + OpenAIInstrumentor().instrument(tracer_provider=provider) + tracer = provider.get_tracer("synthetic-parent") + with tracer.start_as_current_span("parent"): + with client_factory(lambda req: httpx.Response(200, json=chat_response()))( + base_url="http://127.0.0.1:9/v1", api_key="synthetic-model-key" + ) as client: + client.chat.completions.create( + model="bonsai", messages=[{"role": "user", "content": "safe prompt"}] + ) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 2 + parent = next(s for s in spans if s["name"] == "parent") + child = next(s for s in spans if s["name"] == "ChatCompletion") + assert child["parentSpanId"] == parent["spanId"] + assert child["traceId"] == parent["traceId"] + assert attrs(child)["gen_ai.span.kind"] == "LLM" + + +@pytest.mark.parametrize("stream", [False, True]) +def test_async_success_has_one_span(traced, stream): + receiver, provider = traced + + async def run(): + def handler(request): + return ( + httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + content=stream_body(), + ) + if stream + else httpx.Response(200, json=chat_response()) + ) + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-model-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + response = await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe prompt"}], + stream=stream, + ) + if stream: + async for _ in response: + pass + + asyncio.run(run()) + provider.force_flush() + assert len(receiver.spans()) == 1 + assert receiver.spans()[0]["status"]["code"] == "STATUS_CODE_OK" + + +def test_hidden_errors_do_not_export_content(monkeypatch): + for key in ( + "FI_HIDE_INPUTS", + "FI_HIDE_OUTPUTS", + "FI_HIDE_LLM_INVOCATION_PARAMETERS", + ): + monkeypatch.setenv(key, "true") + marker = "synthetic-private-prompt-and-answer" + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing("prismml-error-privacy") + try: + with client_factory( + lambda req: httpx.Response(400, json={"error": {"message": marker}}) + )( + base_url="http://127.0.0.1:9/v1", api_key="synthetic-model-key" + ) as client: + with pytest.raises(openai.BadRequestError): + client.chat.completions.create( + model="bonsai", messages=[{"role": "user", "content": marker}] + ) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + assert marker not in json.dumps(spans) + finally: + OpenAIInstrumentor().uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize( + "body", + [b"", first_chunk(), first_chunk() + b"data: [DONE]\n\n", b"data: [DONE]\n\n"], +) +def test_cli_rejects_missing_terminal_choice(monkeypatch, traced, capsys, body): + _, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + response = httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, content=body + ) + monkeypatch.setattr(app, "OpenAI", client_factory(lambda request: response)) + assert app.main(["--stream"]) == 1 + assert "before a finish reason" in capsys.readouterr().err + assert response.is_closed + + +@pytest.mark.parametrize("reason", ["stop", "length", "tool_calls", "content_filter"]) +@pytest.mark.parametrize("usage_tail", [False, True]) +def test_cli_accepts_terminal_choice_and_usage_tail( + monkeypatch, traced, reason, usage_tail +): + _, provider = traced + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + terminal = { + "id": "synthetic-terminal", + "object": "chat.completion.chunk", + "created": 1, + "model": "bonsai", + "choices": [{"index": 0, "delta": {}, "finish_reason": reason}], + } + body = first_chunk() + ("data: " + json.dumps(terminal) + "\n\n").encode() + if usage_tail: + usage = dict( + terminal, + choices=[], + usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + ) + body += ("data: " + json.dumps(usage) + "\n\n").encode() + body += b"data: [DONE]\n\n" + response = httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, content=body + ) + monkeypatch.setattr(app, "OpenAI", client_factory(lambda request: response)) + assert app.main(["--stream"]) == 0 + assert response.is_closed + + +def test_async_stream_cancellation_finishes_span(traced): + receiver, provider = traced + interruption = asyncio.CancelledError("synthetic stream cancellation") + + class Interrupted(httpx.AsyncByteStream): + async def __aiter__(self): + yield first_chunk() + raise interruption + + async def run(): + async def handler(request): + return httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, stream=Interrupted() + ) + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + stream = await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe"}], + stream=True, + ) + try: + with pytest.raises(asyncio.CancelledError) as caught: + async for _ in stream: + pass + assert caught.value is interruption + finally: + await stream.close() + + asyncio.run(run()) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_async_explicit_close_finishes_span_once(traced): + receiver, provider = traced + + async def run(): + async def handler(request): + return httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + content=stream_body(), + ) + + async with openai.AsyncOpenAI( + base_url="http://127.0.0.1:9/v1", + api_key="synthetic-key", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) as client: + stream = await client.chat.completions.create( + model="bonsai", + messages=[{"role": "user", "content": "safe"}], + stream=True, + ) + await stream.__anext__() + await stream.close() + await stream.close() + + asyncio.run(run()) + provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" diff --git a/python/examples/prismml/tests/test_prismml_recipe.py b/python/examples/prismml/tests/test_prismml_recipe.py index 29be3876a..02c89e7b5 100644 --- a/python/examples/prismml/tests/test_prismml_recipe.py +++ b/python/examples/prismml/tests/test_prismml_recipe.py @@ -25,10 +25,6 @@ FakeOpenAI, chat_response, stream_body, ) -# traceai-openai 0.1.10 and earlier omit gen_ai.request.model on streamed and failed -# calls; later versions (future-agi/traceAI#249) record the requested model there. -# Accept both so this recipe passes on either version. - ROOT = Path(__file__).resolve().parents[1] FI_API_KEY = "placeholder-fi-api-key" FI_SECRET_KEY = "placeholder-fi-secret-key" @@ -267,9 +263,7 @@ def test_stream_accumulation_and_usage(traced, include_usage): assert body.get("stream_options") == ({"include_usage": True} if include_usage else None) span, attrs = one_span(traced) assert attrs["output.value"] == ANSWER - # traceai-openai records the model only from a non-streamed response; flip - # this when the instrumentor records the request model. - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL assert_usage(attrs, USAGE_ATTRIBUTES if include_usage else {}) assert any(event["name"] == "First Token Stream Event" for event in span["events"]) @@ -285,7 +279,7 @@ def test_textless_stream_has_no_raw_response_fallback(traced): assert len(seen) == 1 _, attrs = one_span(traced) assert attrs["output.value"] == "" - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL assert_usage(attrs, {}) @@ -304,9 +298,7 @@ def test_synthetic_401_records_error_without_key(traced): assert len(exceptions) == 1 assert "Synthetic fixture rejection" in json.dumps(exceptions) assert LOCAL_KEY not in json.dumps(span) - # traceai-openai records the model only from a non-streamed response; flip - # this when the instrumentor records the request model. - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL assert_usage(attrs, {}) assert_export_separation(traced[0], traced[2]) @@ -688,7 +680,7 @@ def test_app_subprocess_chat_and_stream(tmp_path, stream, env_project): assert attrs["output.value"] == ANSWER assert json.loads(attrs["gen_ai.request.parameters"])["model"] == REQUEST_MODEL if stream: - assert attrs.get("gen_ai.request.model") in (None, REQUEST_MODEL) + assert attrs["gen_ai.request.model"] == REQUEST_MODEL else: assert attrs["gen_ai.request.model"] == RESPONSE_MODEL assert_usage(attrs, {} if stream else USAGE_ATTRIBUTES) @@ -739,26 +731,63 @@ def test_readme_and_requirements_contract(): if ast.unparse(node.func).startswith("os.environ.") and node.args: if isinstance(node.args[0], ast.Constant): assert node.args[0].value in readme - headings = ["## Install", "## Configure", "## Run", "## Code", - "## What you see in Future AGI", "## Provider specifics", "## Privacy", - "## Limits / not covered", "## Tests"] + headings = ["## Install", "## Start the local server", "## Configure", "## Run", + "## Code", "## View traces", "## Privacy", "## Troubleshooting"] assert [readme.index(heading) for heading in headings] == sorted(readme.index(heading) for heading in headings) - for paths in ("python/examples/prismml/src:python:python/frameworks/openai:python/tests", - "python/examples/prismml/src:python/tests"): - assert f'PYTHONPATH="{paths}"' in readme - assert readme.count("pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs") == 2 - assert "--with 'traceAI-openai==0.1.10'" in readme - assert "--with 'fi-instrumentation-otel==1.1.0'" in readme - assert "TBD" not in readme - assert "## Real-server check" in readme - assert "Ternary-Bonsai-2-2B-PQ2_0.gguf" in readme - assert "The automated contract suite uses a local fake" in readme - assert "not 27B tool calling, vision, or MLX" in readme + for diary in ("## Real-server check", "## Tests", "fixture", "Full suite", "future-agi/traceAI#"): + assert diary not in readme + assert "https://api.futureagi.com/tracer/v1/traces" in readme requirements = (ROOT / "requirements.txt").read_text().splitlines() - assert [line for line in requirements if not line.startswith("#")] == [ - "openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0"] + assert [line for line in requirements if line and not line.startswith("#")] == [ + "openai==3.42.1", "traceAI-openai==0.1.12", "fi-instrumentation-otel==1.2.2"] for path in ROOT.rglob("*"): if path.is_file() and "__pycache__" not in path.parts and path.suffix in {".py", ".md", ".txt"}: text = path.read_text() assert not re.search(r"\b[A-Z]{2,5}-\d{3,}\b", text), str(path) assert not re.search(r"\bsk-[A-Za-z0-9]{16,}\b", text), str(path) + + +def test_readme_code_exports_one_span(monkeypatch, capsys): + """Execute the customer example with real registration and OTLP export.""" + readme = (ROOT / "README.md").read_text() + code = re.search(r"## Code\n.*?```python\n(.*?)```", readme, re.S).group(1) + with Receiver() as receiver, FakeOpenAI() as fake: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("PRISMML_BASE_URL", fake.base_url) + monkeypatch.setenv("FI_PROJECT_NAME", "prismml-readme-example") + namespace = {} + try: + exec(compile(code, str(ROOT / "README.md"), "exec"), namespace) + assert capsys.readouterr().out.strip() == ANSWER + assert len(fake.requests()) == 1 + assert fake.requests()[0]["path"] == "/v1/chat/completions" + spans = receiver.spans() + assert len(spans) == 1 + attrs = attributes(spans[0]) + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.request.model"] == RESPONSE_MODEL + assert attrs["output.value"] == ANSWER + assert_usage(attrs, USAGE_ATTRIBUTES) + assert_export_separation(receiver, "prismml-readme-example") + finally: + OpenAIInstrumentor().uninstrument() + if "provider" in namespace: + namespace["provider"].shutdown() + + +def test_ci_covers_published_source_and_sdk_bounds(): + import yaml + + workflow = ROOT.parents[2] / ".github/workflows/prismml-recipe-tests.yml" + job = yaml.safe_load(workflow.read_text())["jobs"]["recipe"] + cells = job["strategy"]["matrix"]["include"] + expected = { + (packages, python, sdk) + for packages in ("published", "source") + for python, sdk in (("3.10", "1.69.0"), ("3.13", "3.42.1")) + } + assert {(cell["packages"], cell["python"], cell["openai"]) for cell in cells} == expected + installs = "\n".join(step.get("run", "") for step in job["steps"]) + assert 'openai==${{ matrix.openai }}' in installs + assert "pip check" in installs