Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions .github/workflows/prismml-recipe-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,25 @@ permissions:

jobs:
recipe:
name: PrismML recipe (${{ matrix.packages }})
name: PrismML recipe (${{ matrix.packages }}, Python ${{ matrix.python }}, OpenAI ${{ matrix.openai }})
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
packages: [published, source]
include:
- packages: published
python: '3.10'
openai: '1.69.0'
- packages: source
python: '3.10'
openai: '1.69.0'
- packages: published
python: '3.13'
openai: '3.42.1'
- packages: source
python: '3.13'
openai: '3.42.1'
env:
FI_BASE_URL: http://127.0.0.1:9
FI_GRPC_URL: http://127.0.0.1:9
Expand All @@ -35,11 +47,15 @@ jobs:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: '3.11'
python-version: ${{ matrix.python }}
- name: Install published recipe pins and test tools
run: pip install -r python/examples/prismml/requirements.txt pytest httpx 'wrapt<2'
run: pip install -r python/examples/prismml/requirements.txt pytest httpx 'wrapt<2' PyYAML
- name: Select SDK compatibility version
run: pip install 'openai==${{ matrix.openai }}'
- name: Install source distributions
if: matrix.packages == 'source'
run: pip install ./python ./python/frameworks/openai
- name: Check dependency consistency
run: pip check
- name: Loopback-only recipe contracts
run: pytest python/examples/prismml/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs
228 changes: 78 additions & 150 deletions python/examples/prismml/README.md

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions python/examples/prismml/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Installation pins; see README for source verification and the SDK floor.
openai==3.24.0
traceAI-openai==0.1.10
fi-instrumentation-otel==1.1.0
# Published dependencies for the local Bonsai server recipe.
openai==3.42.1
traceAI-openai==0.1.12
fi-instrumentation-otel==1.2.2
17 changes: 14 additions & 3 deletions python/examples/prismml/src/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,21 @@ def main(argv=None) -> int:
stream=args.stream,
)
if args.stream:
for chunk in response:
if chunk.choices:
print(chunk.choices[0].delta.content or "", end="", flush=True)
completed = False
with response:
for chunk in response:
for choice in chunk.choices:
if choice.index == 0:
print(choice.delta.content or "", end="", flush=True)
if choice.finish_reason is not None:
completed = True
print()
# The SDK accepts EOF or [DONE] without a terminal choice. Tracing
# is passive, so the application must reject an incomplete reply.
if not completed:
print("The response ended before a finish reason; retry the request.",
file=sys.stderr)
return 1
else:
print(response.choices[0].message.content or "")
finally:
Expand Down
134 changes: 134 additions & 0 deletions python/examples/prismml/tests/test_prismml_cli_lifecycle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
"""Execute the unmodified PrismML CLI with real loopback HTTP, OTLP, and SIGINT."""

import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest

ROOT = Path(__file__).resolve().parents[1]
from harness import Receiver


@pytest.mark.parametrize(
"mode",
["request_interrupt", "stream_interrupt", "truncated_stream", "premature_done"],
)
def test_real_cli(mode):
request_started = threading.Event()
release = threading.Event()
prefix = {
"id": "synthetic-bonsai",
"object": "chat.completion.chunk",
"created": 1,
"model": "bonsai",
"choices": [
{
"index": 0,
"delta": {"content": "Partial response"},
"finish_reason": None,
}
],
}
frame = ("data: " + json.dumps(prefix) + "\n\n").encode()

if mode == "premature_done":
frame += b"data: [DONE]\n\n"

class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass

def do_POST(self):
self.rfile.read(int(self.headers["Content-Length"]))
request_started.set()
if mode == "request_interrupt":
release.wait(15)
return
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
if mode in ("truncated_stream", "premature_done"):
self.send_header("Content-Length", str(len(frame)))
self.end_headers()
self.wfile.write(frame)
self.wfile.flush()
if mode == "stream_interrupt":
release.wait(15)

server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
proc = None
try:
with Receiver() as receiver:
env = {
k: v
for k, v in os.environ.items()
if not k.startswith(("FI_", "OTEL_", "OPENAI_", "PRISMML_"))
}
env.update(
FI_API_KEY="synthetic-fi-key",
FI_SECRET_KEY="synthetic-fi-secret",
FI_BASE_URL=receiver.origin,
PRISMML_BASE_URL=f"http://127.0.0.1:{server.server_port}/v1",
NO_PROXY="*",
PYTHONDONTWRITEBYTECODE="1",
)
argv = [
sys.executable,
str(ROOT / "src/app.py"),
"--prompt",
"safe synthetic input",
]
if mode != "request_interrupt":
argv.append("--stream")
proc = subprocess.Popen(
argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env
)
assert request_started.wait(10), "CLI never reached the local model server"
first = ""
if mode == "stream_interrupt":
first_char = []
ready = threading.Event()

def consume_one():
first_char.append(proc.stdout.read(1))
ready.set()

reader = threading.Thread(target=consume_one, daemon=True)
reader.start()
assert ready.wait(10), "CLI did not yield the first streamed token"
reader.join(1)
first = first_char[0]
if mode in ("request_interrupt", "stream_interrupt"):
proc.send_signal(signal.SIGINT)
out, err = proc.communicate(timeout=10)
assert proc.returncode != 0, "Incomplete or interrupted CLI must fail"
spans = receiver.spans()
print(
json.dumps(
{
"scenario": mode,
"exit_code": proc.returncode,
"stdout": first + out,
"span_count": len(spans),
"statuses": [s.get("status", {}) for s in spans],
}
)
)
assert len(spans) == 1, f"{mode}: actual CLI exported {len(spans)} spans"
assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR", (
f"{mode}: actual CLI returned {proc.returncode} and reported {spans[0]['status']}"
)
finally:
release.set()
if proc and proc.poll() is None:
proc.kill()
proc.communicate()
server.shutdown()
server.server_close()
thread.join(2)
Loading
Loading