diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 7bf1138c8a8e..e82b9d854aab 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1157,7 +1157,6 @@ json oaicompat_chat_params_parse( auto tools = json_value(body, "tools", json()); auto has_tools = tools.is_array() && !tools.empty(); - auto stream = json_value(body, "stream", false); auto tool_choice = json_value(body, "tool_choice", std::string("auto")); if (!opt.use_jinja) { @@ -1401,11 +1400,10 @@ json oaicompat_chat_params_parse( } // Handle "logprobs" field - // TODO: The response format of this option is not yet OAI-compatible, but seems like no one really using it; We may need to fix it in the future + // Response is OpenAI-modern shape (choices[].logprobs.content[]); each entry also carries the + // integer token id. With tools, the streaming path emits per-token logprobs for all tokens + // (including tool-call tokens), matching the non-streaming path. if (json_value(body, "logprobs", false)) { - if (has_tools && stream) { - throw std::invalid_argument("logprobs is not supported with tools + stream"); - } llama_params["n_probs"] = json_value(body, "top_logprobs", 20); } else if (body.contains("top_logprobs") && !body.at("top_logprobs").is_null()) { throw std::invalid_argument("top_logprobs requires logprobs to be set to true"); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index b6835e43459e..71a0664f3144 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1869,13 +1869,22 @@ struct server_context_impl { slot.n_sent_text += result.text_to_send.size(); // add the token to slot queue and cache } else { + // stop sequence matched: exclude this token from both the content stream + // and its own logprobs entry, so neither leaks text past the stop boundary result.text_to_send = ""; + result.txt = ""; } + } else { + // withhold content until the char completes; result.txt keeps this token's + // own piece so its logprobs entry isn't dropped along with the content + result.text_to_send = ""; + } - slot.add_token(result); - if (slot.task->params.stream) { - send_partial_response(slot, result, false); - } + // every sampled token gets its own logprobs entry (and, if streaming, its own + // partial chunk) regardless of whether its content is visible yet - see result.txt + slot.add_token(result); + if (slot.task->params.stream) { + send_partial_response(slot, result, false); } if (incomplete) { @@ -3876,6 +3885,7 @@ struct server_context_impl { completion_token_output result; result.tok = id; result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok)); + result.txt = result.text_to_send; result.prob = 1.0f; // TODO: set it here instead of doing inside populate_token_probs if (slot.task->params.sampling.n_probs > 0) { @@ -4002,6 +4012,7 @@ struct server_context_impl { result.tok = ids[i]; result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok)); + result.txt = result.text_to_send; result.prob = 1.0f; // set later // TODO: set result.probs diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 0d3beb313cea..6e3a6632ae3c 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -282,12 +282,12 @@ json completion_token_output::to_json(bool post_sampling_probs) const { json completion_token_output::probs_vector_to_json(const std::vector & probs, bool post_sampling_probs) { json out = json::array(); for (const auto & p : probs) { - std::string txt(p.text_to_send); + std::string txt(p.txt); txt.resize(validate_utf8(txt)); out.push_back(json { {"id", p.tok}, {"token", txt}, - {"bytes", str_to_bytes(p.text_to_send)}, + {"bytes", str_to_bytes(p.txt)}, { post_sampling_probs ? "prob" : "logprob", post_sampling_probs ? p.prob : logarithm(p.prob) @@ -483,6 +483,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() { {"object", "chat.completion.chunk"}, }); } + // index of the last diff chunk, if any, before the finish-reason chunk below is pushed + size_t last_diff_idx = deltas.empty() ? SIZE_MAX : deltas.size() - 1; deltas.push_back({ {"choices", json::array({ @@ -499,6 +501,15 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() { {"object", "chat.completion.chunk"}, }); + // attach any leftover pending_probs to the last diff chunk, or the finish-reason + // chunk if there was no diff at all + if (!probs_to_emit.empty()) { + size_t target_idx = (last_diff_idx != SIZE_MAX) ? last_diff_idx : deltas.size() - 1; + deltas[target_idx].at("choices").at(0)["logprobs"] = json { + {"content", completion_token_output::probs_vector_to_json(probs_to_emit, post_sampling_probs)}, + }; + } + if (include_usage) { // OpenAI API spec for chat.completion.chunks specifies an empty `choices` array for the last chunk when including usage // https://platform.openai.com/docs/api-reference/chat_streaming/streaming#chat_streaming/streaming-choices @@ -992,6 +1003,15 @@ void server_task_result_cmpl_partial::update(task_result_state & state) { } state.update_chat_msg(content, true, oaicompat_msg_diffs); + // queue this token's logprobs, flush once a chunk has visible delta content to attach them to + if (prob_output.probs.size() > 0) { + state.pending_probs.push_back(prob_output); + } + if (is_progress || n_decoded == 1 || !oaicompat_msg_diffs.empty()) { + probs_to_emit = std::move(state.pending_probs); + state.pending_probs.clear(); + } + // Copy current state for use in to_json_*() (reflects state BEFORE this chunk) thinking_block_started = state.thinking_block_started; text_block_started = state.text_block_started; @@ -1146,9 +1166,9 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() { auto & last_json = deltas[deltas.size() - 1]; GGML_ASSERT(last_json.at("choices").size() >= 1); - if (prob_output.probs.size() > 0) { + if (!probs_to_emit.empty()) { last_json.at("choices").at(0)["logprobs"] = json { - {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)}, + {"content", completion_token_output::probs_vector_to_json(probs_to_emit, post_sampling_probs)}, }; } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 9c99143f8e19..5a7ef1205f59 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -101,6 +101,8 @@ struct task_params { json to_json(bool only_metrics = false) const; }; +struct completion_token_output; // defined below, used by task_result_state + // struct for tracking the state of a task (e.g., for streaming) struct task_result_state { // tracking diffs for partial tool calls @@ -111,6 +113,10 @@ struct task_result_state { std::vector generated_tool_call_ids; std::unordered_set sent_tool_call_names; + // logprobs for tokens whose text hasn't produced a chat-msg diff yet (still + // being buffered by the parser); flushed onto the next chunk that does + std::vector pending_probs; + // for OpenAI Responses and Anthropic streaming API: // track output item / content block state across chunks bool thinking_block_started = false; @@ -300,6 +306,10 @@ struct completion_token_output { llama_token tok; float prob; std::string text_to_send; + // this token's own decoded piece, for logprobs attribution - independent of + // text_to_send, which can be withheld (incomplete UTF-8), merged with earlier + // tokens (completing a multi-token UTF-8 char), or blanked (matched stop word) + std::string txt; struct prob_info { llama_token tok; std::string txt; @@ -337,6 +347,7 @@ struct server_task_result_cmpl_final : server_task_result { bool post_sampling_probs; std::vector probs_output; + std::vector probs_to_emit; // leftover pending_probs flushed by update() std::vector response_fields; task_params generation_params; @@ -366,6 +377,10 @@ struct server_task_result_cmpl_final : server_task_result { is_updated = true; oaicompat_msg = state.update_chat_msg(content, false, oaicompat_msg_diffs); + // last update for the request: drain pending_probs unconditionally + probs_to_emit = std::move(state.pending_probs); + state.pending_probs.clear(); + oai_resp_id = state.oai_resp_id; oai_resp_reasoning_id = state.oai_resp_reasoning_id; oai_resp_message_id = state.oai_resp_message_id; @@ -405,6 +420,7 @@ struct server_task_result_cmpl_partial : server_task_result { bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream) // ref: https://github.com/ggml-org/llama.cpp/pull/23884 completion_token_output prob_output; + std::vector probs_to_emit; // pending_probs flushed by update() for this chunk server_slot_stats stats; result_prompt_progress progress; diff --git a/tools/server/tests/unit/test_tool_call.py b/tools/server/tests/unit/test_tool_call.py index 87c4ad166c38..4684bc15b9ec 100755 --- a/tools/server/tests/unit/test_tool_call.py +++ b/tools/server/tests/unit/test_tool_call.py @@ -61,6 +61,28 @@ class ToolDefinition(TypedDict): ), ) +# forces (via grammar-constrained decoding, same trick as TEST_TOOL's boolean "const" above) +# a tool argument containing a character that spans multiple tokens in UTF-8, regardless of +# model quality - used to reproduce the dropped/misattributed logprobs bug for such characters +MULTIBYTE_CHAR = "\U0001F9EA" # test tube emoji, 4 UTF-8 bytes, multiple BPE tokens +TEST_TOOL_MULTIBYTE = ToolDefinition( + type = "function", + function = ToolFunction( + name = "test", + description = "", + parameters = ToolParameters( + type = "object", + properties = { + "note": { + "type": "string", + "const": f"ok {MULTIBYTE_CHAR} done", + }, + }, + required = ["note"], + ), + ), +) + PYTHON_TOOL = ToolDefinition( type = "function", function = ToolFunction( @@ -318,6 +340,210 @@ def test_completion_without_tool_call_fast(template_name: str, n_predict: int, t do_test_completion_without_tool_call(server, n_predict, tools, tool_choice, stream=stream == CompletionMode.STREAMED) +def do_test_logprobs_with_tools(server: ServerProcess, stream: bool, top_logprobs: int): + def assert_valid_bytes(d: dict): + # bytes can legitimately be [] for a special/control token that decodes to no + # text (e.g. id 128010 below) - only the container shape and element range matter + assert "bytes" in d and isinstance(d["bytes"], list) + assert all(isinstance(b, int) and 0 <= b <= 255 for b in d["bytes"]) + + def check_logprob_entries(entries: list[dict]): + for entry in entries: + assert "id" in entry and isinstance(entry["id"], int) + assert "token" in entry + assert entry["logprob"] <= 0.0 + assert_valid_bytes(entry) + assert len(entry["top_logprobs"]) == top_logprobs + for top in entry["top_logprobs"]: + assert "id" in top and isinstance(top["id"], int) + assert "token" in top + assert top["logprob"] <= 0.0 + assert_valid_bytes(top) + + body = { + "max_tokens": 128, + "messages": [ + {"role": "system", "content": "You are a coding assistant."}, + {"role": "user", "content": "Write an example"}, + ], + "tools": [TEST_TOOL], + "tool_choice": "required", + "parallel_tool_calls": False, + "stream": stream, + "logprobs": True, + "top_logprobs": top_logprobs, + "temperature": 0.0, + "top_k": 1, + "top_p": 1.0, + } + + if stream: + body["stream_options"] = {"include_usage": True} + tool_calls: list[dict] = [] + finish_reason = None + total_logprob_entries = 0 + completion_tokens = None + for chunk in server.make_stream_request("POST", "/v1/chat/completions", data=body): + if chunk.get("usage"): + completion_tokens = chunk["usage"]["completion_tokens"] + if not chunk["choices"]: + continue + choice = chunk["choices"][0] + if choice.get("finish_reason") is not None: + finish_reason = choice["finish_reason"] + for tc in choice["delta"].get("tool_calls", []): + if tc["index"] >= len(tool_calls): + tool_calls.append(dict(function=dict(name="", arguments=""))) + fct = tc["function"] + if fct.get("name") is not None: + tool_calls[tc["index"]]["function"]["name"] += fct["name"] + if fct.get("arguments") is not None: + tool_calls[tc["index"]]["function"]["arguments"] += fct["arguments"] + logprobs = choice.get("logprobs") + if logprobs: + total_logprob_entries += len(logprobs["content"]) + check_logprob_entries(logprobs["content"]) + assert completion_tokens is not None, "Expected a usage chunk reporting completion_tokens" + # token completeness: every generated token must carry logprobs, not just + # whichever ones happened to land on a chunk with visible delta content + assert total_logprob_entries == completion_tokens + else: + resp = server.make_any_request("POST", "/v1/chat/completions", data=body) + choice = resp["choices"][0] + finish_reason = choice["finish_reason"] + tool_calls = choice["message"].get("tool_calls") or [] + logprobs = choice.get("logprobs") + assert logprobs and logprobs["content"], f'Expected logprobs in {choice}' + check_logprob_entries(logprobs["content"]) + + assert finish_reason == "tool_calls", f"Expected a tool call, got finish_reason={finish_reason}" + assert len(tool_calls) == 1, f"Expected exactly 1 tool call, got {tool_calls}" + tool_call = tool_calls[0] + assert tool_call["function"]["name"] == TEST_TOOL["function"]["name"] + actual_arguments = json.loads(tool_call["function"]["arguments"]) + assert actual_arguments.get("success") is True, f"Unexpected arguments: {actual_arguments}" + + +@pytest.mark.slow +@pytest.mark.parametrize("top_logprobs", [5, 100]) +@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED]) +def test_logprobs_with_tools(stream: CompletionMode, top_logprobs: int): + # Regression: tools + stream + logprobs used to be rejected with a 400. It is now allowed and, + # matching the non-streaming path, emits per-token logprobs (including tool-call tokens) in the + # OpenAI-modern shape, retaining the integer token id. Per-token text can be split across + # `content` and `tool_calls` (or be parser-internal scaffolding with no visible text at all), + # so stream completeness is checked by count against the server's own `usage.completion_tokens`, + # rather than by reconstructing a single output string. + # + # This needs a real model: with `tool_choice: "required"`, the tiny test model used elsewhere + # in this file is not guaranteed to ever actually emit a tool call (see the disabled + # test_completion_with_required_tool_tiny_fast above, PR #22654), so it can't exercise the + # tool-call-argument buffering path this test is meant to catch. + global server + server.offline = False # not one of the ServerPreset models pre-fetched by load_all() + server.jinja = True + server.n_ctx = 8192 + server.n_predict = 128 + server.model_hf_repo = "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M" + server.model_hf_file = None + server.chat_template_file = '../../../models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja' + server.start(timeout_seconds=TIMEOUT_START_SLOW) + do_test_logprobs_with_tools(server, stream == CompletionMode.STREAMED, top_logprobs) + + +def do_test_logprobs_multibyte_char(server: ServerProcess, stream: bool): + body = { + "max_tokens": 128, + "messages": [ + {"role": "system", "content": "You are a coding assistant."}, + {"role": "user", "content": "Write an example"}, + ], + "tools": [TEST_TOOL_MULTIBYTE], + "tool_choice": "required", + "parallel_tool_calls": False, + "stream": stream, + "logprobs": True, + "top_logprobs": 5, + "temperature": 0.0, + "top_k": 1, + "top_p": 1.0, + } + + # bytes from every logprobs entry, in emission order - reconstructing the generated + # text from these (rather than content/tool_calls deltas) catches dropped or + # misattributed bytes for multi-token UTF-8 characters + all_bytes = bytearray() + tool_calls: list[dict] = [] + finish_reason = None + + if stream: + body["stream_options"] = {"include_usage": True} + total_logprob_entries = 0 + completion_tokens = None + for chunk in server.make_stream_request("POST", "/v1/chat/completions", data=body): + if chunk.get("usage"): + completion_tokens = chunk["usage"]["completion_tokens"] + if not chunk["choices"]: + continue + choice = chunk["choices"][0] + if choice.get("finish_reason") is not None: + finish_reason = choice["finish_reason"] + for tc in choice["delta"].get("tool_calls", []): + if tc["index"] >= len(tool_calls): + tool_calls.append(dict(function=dict(name="", arguments=""))) + fct = tc["function"] + if fct.get("name") is not None: + tool_calls[tc["index"]]["function"]["name"] += fct["name"] + if fct.get("arguments") is not None: + tool_calls[tc["index"]]["function"]["arguments"] += fct["arguments"] + logprobs = choice.get("logprobs") + if logprobs: + total_logprob_entries += len(logprobs["content"]) + for entry in logprobs["content"]: + all_bytes.extend(entry["bytes"]) + assert completion_tokens is not None, "Expected a usage chunk reporting completion_tokens" + assert total_logprob_entries == completion_tokens + else: + resp = server.make_any_request("POST", "/v1/chat/completions", data=body) + choice = resp["choices"][0] + finish_reason = choice["finish_reason"] + tool_calls = choice["message"].get("tool_calls") or [] + logprobs = choice.get("logprobs") + assert logprobs and logprobs["content"], f'Expected logprobs in {choice}' + for entry in logprobs["content"]: + all_bytes.extend(entry["bytes"]) + assert len(logprobs["content"]) == resp["usage"]["completion_tokens"] + + assert finish_reason == "tool_calls", f"Expected a tool call, got finish_reason={finish_reason}" + assert len(tool_calls) == 1, f"Expected exactly 1 tool call, got {tool_calls}" + tool_call = tool_calls[0] + actual_arguments = json.loads(tool_call["function"]["arguments"]) + assert actual_arguments.get("note") == f"ok {MULTIBYTE_CHAR} done", f"Unexpected arguments: {actual_arguments}" + + # the logprobs bytes, reconstructed in order, must contain the multi-byte character + # exactly once, with no bytes lost (would fail to decode or omit the character) and + # none duplicated (would decode to more than one occurrence of the character) + reconstructed = bytes(all_bytes).decode("utf-8") + assert reconstructed.count(MULTIBYTE_CHAR) == 1, f"Expected exactly one occurrence of {MULTIBYTE_CHAR!r} in logprobs-reconstructed text, got: {reconstructed!r}" + + +@pytest.mark.slow +@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED]) +def test_logprobs_multibyte_tool_arg(stream: CompletionMode): + # Regression: a multi-token UTF-8 character in a tool-call argument used to drop + # logprobs entries for its earlier token(s). Needs a real model, same as above. + global server + server.offline = False # not one of the ServerPreset models pre-fetched by load_all() + server.jinja = True + server.n_ctx = 8192 + server.n_predict = 128 + server.model_hf_repo = "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M" + server.model_hf_file = None + server.chat_template_file = '../../../models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja' + server.start(timeout_seconds=TIMEOUT_START_SLOW) + do_test_logprobs_multibyte_char(server, stream == CompletionMode.STREAMED) + + @pytest.mark.slow @pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED]) @pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [