llama-server : make chat logprobs response OpenAI-compatible

tools/server/server-common.cpp

@@ -1157,7 +1157,6 @@ json oaicompat_chat_params_parse(
 
     auto tools = json_value(body, "tools", json());
     auto has_tools = tools.is_array() && !tools.empty();
-    auto stream = json_value(body, "stream", false);
     auto tool_choice = json_value(body, "tool_choice", std::string("auto"));
 
     if (!opt.use_jinja) {
@@ -1401,11 +1400,10 @@ json oaicompat_chat_params_parse(
     }
 
     // Handle "logprobs" field
-    // TODO: The response format of this option is not yet OAI-compatible, but seems like no one really using it; We may need to fix it in the future
+    // Response is OpenAI-modern shape (choices[].logprobs.content[]); each entry also carries the
+    // integer token id. With tools, the streaming path emits per-token logprobs for all tokens
+    // (including tool-call tokens), matching the non-streaming path.
     if (json_value(body, "logprobs", false)) {
-        if (has_tools && stream) {
-            throw std::invalid_argument("logprobs is not supported with tools + stream");
-        }
         llama_params["n_probs"] = json_value(body, "top_logprobs", 20);
     } else if (body.contains("top_logprobs") && !body.at("top_logprobs").is_null()) {
         throw std::invalid_argument("top_logprobs requires logprobs to be set to true");

tools/server/server-context.cpp

@@ -1869,13 +1869,22 @@ struct server_context_impl {
                 slot.n_sent_text += result.text_to_send.size();
                 // add the token to slot queue and cache
             } else {
+                // stop sequence matched: exclude this token from both the content stream
+                // and its own logprobs entry, so neither leaks text past the stop boundary
                 result.text_to_send = "";
+                result.txt          = "";
             }
+        } else {
+            // withhold content until the char completes; result.txt keeps this token's
+            // own piece so its logprobs entry isn't dropped along with the content
+            result.text_to_send = "";
+        }
 
-            slot.add_token(result);
-            if (slot.task->params.stream) {
-                send_partial_response(slot, result, false);
-            }
+        // every sampled token gets its own logprobs entry (and, if streaming, its own
+        // partial chunk) regardless of whether its content is visible yet - see result.txt
+        slot.add_token(result);
+        if (slot.task->params.stream) {
+            send_partial_response(slot, result, false);
         }
 
         if (incomplete) {
@@ -3876,6 +3885,7 @@ struct server_context_impl {
             completion_token_output result;
             result.tok          = id;
             result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok));
+            result.txt          = result.text_to_send;
             result.prob         = 1.0f; // TODO: set it here instead of doing inside populate_token_probs
 
             if (slot.task->params.sampling.n_probs > 0) {
@@ -4002,6 +4012,7 @@ struct server_context_impl {
 
                 result.tok          = ids[i];
                 result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok));
+                result.txt          = result.text_to_send;
                 result.prob         = 1.0f; // set later
 
                 // TODO: set result.probs

tools/server/server-task.cpp

@@ -282,12 +282,12 @@ json completion_token_output::to_json(bool post_sampling_probs) const {
 json completion_token_output::probs_vector_to_json(const std::vector<completion_token_output> & probs, bool post_sampling_probs) {
     json out = json::array();
     for (const auto & p : probs) {
-        std::string txt(p.text_to_send);
+        std::string txt(p.txt);
         txt.resize(validate_utf8(txt));
         out.push_back(json {
             {"id",           p.tok},
             {"token",        txt},
-            {"bytes",        str_to_bytes(p.text_to_send)},
+            {"bytes",        str_to_bytes(p.txt)},
             {
                 post_sampling_probs ? "prob" : "logprob",
                 post_sampling_probs ? p.prob : logarithm(p.prob)
@@ -483,6 +483,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
             {"object", "chat.completion.chunk"},
         });
     }
+    // index of the last diff chunk, if any, before the finish-reason chunk below is pushed
+    size_t last_diff_idx = deltas.empty() ? SIZE_MAX : deltas.size() - 1;
 
     deltas.push_back({
         {"choices", json::array({
@@ -499,6 +501,15 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
         {"object",             "chat.completion.chunk"},
     });
 
+    // attach any leftover pending_probs to the last diff chunk, or the finish-reason
+    // chunk if there was no diff at all
+    if (!probs_to_emit.empty()) {
+        size_t target_idx = (last_diff_idx != SIZE_MAX) ? last_diff_idx : deltas.size() - 1;
+        deltas[target_idx].at("choices").at(0)["logprobs"] = json {
+            {"content", completion_token_output::probs_vector_to_json(probs_to_emit, post_sampling_probs)},
+        };
+    }
+
     if (include_usage) {
         // OpenAI API spec for chat.completion.chunks specifies an empty `choices` array for the last chunk when including usage
         // https://platform.openai.com/docs/api-reference/chat_streaming/streaming#chat_streaming/streaming-choices
@@ -992,6 +1003,15 @@ void server_task_result_cmpl_partial::update(task_result_state & state) {
     }
     state.update_chat_msg(content, true, oaicompat_msg_diffs);
 
+    // queue this token's logprobs, flush once a chunk has visible delta content to attach them to
+    if (prob_output.probs.size() > 0) {
+        state.pending_probs.push_back(prob_output);
+    }
+    if (is_progress || n_decoded == 1 || !oaicompat_msg_diffs.empty()) {
+        probs_to_emit = std::move(state.pending_probs);
+        state.pending_probs.clear();
+    }
+
     // Copy current state for use in to_json_*() (reflects state BEFORE this chunk)
     thinking_block_started = state.thinking_block_started;
     text_block_started     = state.text_block_started;
@@ -1146,9 +1166,9 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
         auto & last_json = deltas[deltas.size() - 1];
         GGML_ASSERT(last_json.at("choices").size() >= 1);
 
-        if (prob_output.probs.size() > 0) {
+        if (!probs_to_emit.empty()) {
             last_json.at("choices").at(0)["logprobs"] = json {
-                {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)},
+                {"content", completion_token_output::probs_vector_to_json(probs_to_emit, post_sampling_probs)},
             };
         }
 

tools/server/server-task.h

@@ -101,6 +101,8 @@ struct task_params {
     json to_json(bool only_metrics = false) const;
 };
 
+struct completion_token_output; // defined below, used by task_result_state
+
 // struct for tracking the state of a task (e.g., for streaming)
 struct task_result_state {
     // tracking diffs for partial tool calls
@@ -111,6 +113,10 @@ struct task_result_state {
     std::vector<std::string> generated_tool_call_ids;
     std::unordered_set<size_t> sent_tool_call_names;
 
+    // logprobs for tokens whose text hasn't produced a chat-msg diff yet (still
+    // being buffered by the parser); flushed onto the next chunk that does
+    std::vector<completion_token_output> pending_probs;
+
     // for OpenAI Responses and Anthropic streaming API:
     // track output item / content block state across chunks
     bool thinking_block_started = false;
@@ -300,6 +306,10 @@ struct completion_token_output {
     llama_token tok;
     float prob;
     std::string text_to_send;
+    // this token's own decoded piece, for logprobs attribution - independent of
+    // text_to_send, which can be withheld (incomplete UTF-8), merged with earlier
+    // tokens (completing a multi-token UTF-8 char), or blanked (matched stop word)
+    std::string txt;
     struct prob_info {
         llama_token tok;
         std::string txt;
@@ -337,6 +347,7 @@ struct server_task_result_cmpl_final : server_task_result {
 
     bool post_sampling_probs;
     std::vector<completion_token_output> probs_output;
+    std::vector<completion_token_output> probs_to_emit; // leftover pending_probs flushed by update()
     std::vector<std::string>  response_fields;
 
     task_params generation_params;
@@ -366,6 +377,10 @@ struct server_task_result_cmpl_final : server_task_result {
         is_updated = true;
         oaicompat_msg = state.update_chat_msg(content, false, oaicompat_msg_diffs);
 
+        // last update for the request: drain pending_probs unconditionally
+        probs_to_emit = std::move(state.pending_probs);
+        state.pending_probs.clear();
+
         oai_resp_id = state.oai_resp_id;
         oai_resp_reasoning_id = state.oai_resp_reasoning_id;
         oai_resp_message_id = state.oai_resp_message_id;
@@ -405,6 +420,7 @@ struct server_task_result_cmpl_partial : server_task_result {
     bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
                            // ref: https://github.com/ggml-org/llama.cpp/pull/23884
     completion_token_output prob_output;
+    std::vector<completion_token_output> probs_to_emit; // pending_probs flushed by update() for this chunk
     server_slot_stats stats;
     result_prompt_progress progress;