From 171210ee89e3944d1c606d50ad779a5dfb1651b8 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sun, 4 Oct 2026 11:47:16 +0100 Subject: [PATCH] Increase neural reply deadline to two seconds --- neural/README.md | 2 +- neural/src/lib.rs | 4 +++- neural/src/process.rs | 9 ++++++--- neural/tests/fake_worker.rs | 8 +++++++- neural/tests/lifecycle.rs | 20 ++++++++++++++++++++ 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/neural/README.md b/neural/README.md index 5c969a4..9c6485f 100644 --- a/neural/README.md +++ b/neural/README.md @@ -6,7 +6,7 @@ This candidate is not yet production-qualified. The optimized Windows worker pas The fixed policy reranks eight statistical candidates using SmolLM2-135M Q8. It scores every token of a word plus the probability of a following word boundary, uses the current sentence capped at 64 tokens and returns at most five ordered words. Scores from different models are never combined or exposed as shared probabilities. Limits above five are errors. Minimum grapheme settings are honored; unigram-only requests stay statistical. The same caller-owned predictor supplies immediate suggestions and the shortlist, including its current personal snapshot. -The controller returns immediate words synchronously. A persistent child loads the model once and refines through private bounded pipes. There is one active request and one replaceable pending request. `poll()` yields only the latest request; callers should also match its request ID before displaying it. Session changes and `reset()` invalidate old results and clear the worker's context. Loading is separate from the 500 ms inference deadline. Startup has a 30 second bound. Failure stops the worker and leaves immediate suggestions available. Recovery requires `retry()`. `shutdown()` kills and joins the worker and clears pending text. +The controller returns immediate words synchronously. A persistent child loads the model once and refines through private bounded pipes. There is one active request and one replaceable pending request. `poll()` yields only the latest request; callers should also match its request ID before displaying it. Session changes and `reset()` invalidate old results and clear the worker's context. Loading is separate from the 2 second reply deadline for inference and reset. Startup has a 30 second bound. Failure stops the worker and leaves immediate suggestions available. Recovery requires `retry()`. `shutdown()` kills and joins the worker and clears pending text. ## Build diff --git a/neural/src/lib.rs b/neural/src/lib.rs index 61fcfb7..0f66a42 100644 --- a/neural/src/lib.rs +++ b/neural/src/lib.rs @@ -14,6 +14,8 @@ use std::{ }; use switchify_prediction::{Options, Predictor, normalize, sentences}; +const REPLY_DEADLINE_MS: u64 = 2_000; + pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -144,7 +146,7 @@ impl Refiner { threads: config.threads, context_tokens: 64, shortlist: 8, - deadline_ms: 500, + deadline_ms: REPLY_DEADLINE_MS, }; let state = Arc::new(( Mutex::new(Shared { diff --git a/neural/src/process.rs b/neural/src/process.rs index 5f8b476..8e8a6fc 100644 --- a/neural/src/process.rs +++ b/neural/src/process.rs @@ -1,5 +1,5 @@ use crate::{ - Config, Failure, Refined, State, Status, pause, + Config, Failure, REPLY_DEADLINE_MS, Refined, State, Status, pause, protocol::{self, Command, Reply}, }; use std::{ @@ -149,7 +149,10 @@ pub(super) fn run(config: Config, accelerated: bool, state: State) { if let Some(w) = &mut worker { if reset { w.send(Command::Reset)?; - if !matches!(w.receive(&state, Duration::from_millis(500))?, Reply::Reset) { + if !matches!( + w.receive(&state, Duration::from_millis(REPLY_DEADLINE_MS))?, + Reply::Reset + ) { return Err(Failure::Protocol); } } @@ -161,7 +164,7 @@ pub(super) fn run(config: Config, accelerated: bool, state: State) { let limit = query.limit.min(query.candidates.len()); let candidates = query.candidates.clone(); w.send(Command::Predict(query))?; - match w.receive(&state, Duration::from_millis(500))? { + match w.receive(&state, Duration::from_millis(REPLY_DEADLINE_MS))? { Reply::Ranked { id: actual, words, diff --git a/neural/tests/fake_worker.rs b/neural/tests/fake_worker.rs index fc92ee1..6e37ea1 100644 --- a/neural/tests/fake_worker.rs +++ b/neural/tests/fake_worker.rs @@ -29,12 +29,18 @@ fn main() { let mut input = io::stdin().lock(); while let Ok(command) = protocol::read_frame(&mut input) { let reply = match command { - Command::Reset => Reply::Reset, + Command::Reset => { + if mode == "slow-reset" { + thread::sleep(Duration::from_millis(750)); + } + Reply::Reset + } Command::Predict(mut query) => { match mode.as_str() { "crash" => return, "stall" => thread::sleep(Duration::from_secs(30)), "delay" => thread::sleep(Duration::from_millis(100)), + "slow" => thread::sleep(Duration::from_millis(750)), "malformed" => { out.write_all(&[1, 0, 0, 0, b'!']).unwrap(); out.flush().unwrap(); diff --git a/neural/tests/lifecycle.rs b/neural/tests/lifecycle.rs index b9257c2..3457648 100644 --- a/neural/tests/lifecycle.rs +++ b/neural/tests/lifecycle.rs @@ -121,6 +121,26 @@ fn immediate_snapshot_options_and_latest_only() { ["can't"] ); } +#[test] +fn replies_beyond_half_a_second_refine_without_blocking_the_snapshot() { + for mode in ["slow", "slow-reset"] { + let (_temp, predictor, mut engine) = fixture(mode); + until(|| engine.status() == Status::Ready); + assert_eq!(engine.capabilities().deadline_ms, 2_000); + let immediate = engine.submit(&predictor, "", "", options(), 1).unwrap(); + assert_eq!(immediate.words.len(), 5); + assert!(immediate.refinement_requested); + assert!(engine.poll().is_none()); + let mut refined = None; + until(|| { + refined = engine.poll(); + refined.is_some() + }); + assert_eq!(refined.unwrap().request_id, immediate.request_id); + assert_eq!(engine.status(), Status::Ready); + } +} + #[test] fn failures_leave_immediate_available_and_require_explicit_retry() { for mode in [