From 6c77b045985e8362c284eedd53a129704693f72b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 6 Oct 2026 23:15:19 -0500 Subject: [PATCH 1/2] train: the training context keeps no recurrent rollback slots, and the exact walk refuses them by name, never asserts Kimi's first exact run on the 5090 (2026-10-07 04:09Z, 27B hybrid, 66,560-token window) took the SERVING process down: server-train builds its context from the serving params, which carry n_rs_seq (recurrent-state snapshots for the MTP draft's rollback), and the delta-net helper's "one state per sequence" invariant asserted mid-window. Training runs in the serving process, so an assert there is an outage. - server-train: the training context sets n_rs_seq = 0 (training never rolls back). - opt_epoch_iter: a recurrent context with rollback slots refuses the exact walk by name before anything runs (the assert stays as the unreachable invariant). - the state restore also restores the cell's position, so the next chunk reads consecutive (the crash log's "non-consecutive token position 65640 after 66345"). test-walk-exact, regression: a hybrid context with n_rs_seq = 4. Without the fix it aborts at delta-net-base.cpp:499 (the 5090's assert); with it, it refuses by name and the adapter is untouched. 1.5B and 0.8B results unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- src/llama-context.cpp | 20 ++++++++++++++++++- tests/test-walk-exact.cpp | 36 +++++++++++++++++++++++++++++++++++ tools/server/server-train.cpp | 5 +++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 2ffe62e3d52b..c55fb2b8ad09 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3758,6 +3758,17 @@ void llama_context::opt_epoch_iter( // Each window starts from the REQUESTED horizon: a halving that fit one window's device // budget must not shrink every later window and epoch, which may be shorter (Fable). opt_walk_horizon = opt_walk_horizon_req; + // A recurrent context with rollback slots keeps several states per sequence; the walk + // carries exactly one. Refused by name, never asserted: training runs inside the serving + // process, and an assert there takes serving down (2026-10-07 04:09Z on the 5090). + if (recr != nullptr && cparams.n_rs_seq > 0) { + opt_failure = "the exact walk needs a training context with no recurrent rollback slots (n_rs_seq = 0, it has " + + std::to_string(cparams.n_rs_seq) + "): create the training context with n_rs_seq = 0"; + LLAMA_LOG_ERROR("%s: %s\n", __func__, opt_failure.c_str()); + opt_alloc_failed.store(true); + opt_stop_requested.store(true); + return; + } // A recurrent model's state is carried the same way, one chunk at a time: the state // entering a chunk is a GRAD leaf, the state it leaves meets the next chunk's gradient // (build_rs, build_walk_state_exit). The reverse pass restores each chunk's entry state @@ -3856,19 +3867,26 @@ void llama_context::opt_epoch_iter( // snapshotted at each chunk's start (empty before the first: it starts from zero) struct state_snapshot { bool empty = true; - std::vector> r, s; // per layer + llama_pos pos = -1; // the cell's last position at the snapshot + std::vector> r, s; // per layer }; std::vector snaps(recr != nullptr ? chunks.size() : 0); auto state_rows = [&](bool write, state_snapshot & snap) { const int32_t cell = recr->cells[0].tail; // the cell holding sequence 0's state if (!write) { snap.empty = cell < 0; + snap.pos = cell < 0 ? -1 : recr->cells[cell].pos; snap.r.assign(recr->r_l.size(), {}); snap.s.assign(recr->s_l.size(), {}); } if (cell < 0) { return; } + // the restored state is the state AFTER snap.pos, so the cell says so: left at the + // window's end, the next ubatch read as non-consecutive ("position 65640 after 66345") + if (write) { + recr->cells[cell].pos = snap.pos; + } for (size_t il = 0; il < recr->r_l.size(); ++il) { for (int kind = 0; kind < 2; ++kind) { ggml_tensor * t = kind == 0 ? recr->r_l[il] : recr->s_l[il]; diff --git a/tests/test-walk-exact.cpp b/tests/test-walk-exact.cpp index 958c14ae0063..0487f90f44ab 100644 --- a/tests/test-walk-exact.cpp +++ b/tests/test-walk-exact.cpp @@ -41,6 +41,8 @@ static ggml_opt_optimizer_params sgd_pars(void *) { // q8_0, in a flash-attention context that stores V untransposed) measure the straight-through // approximation the walk makes there (Fable on #47: the gradient is taken at the cache's values) static ggml_type g_cache_type = GGML_TYPE_F32; +// recurrent rollback slots in the context (serving keeps some for speculative decoding) +static uint32_t g_n_rs_seq = 0; static llama_context * make_ctx(const common_params & params, llama_model * model, uint32_t chunk) { auto cparams = common_context_params_to_llama(params); @@ -54,6 +56,7 @@ static llama_context * make_ctx(const common_params & params, llama_model * mode cparams.flash_attn_type = g_cache_type == GGML_TYPE_F32 ? LLAMA_FLASH_ATTN_TYPE_DISABLED : LLAMA_FLASH_ATTN_TYPE_ENABLED; cparams.type_k = g_cache_type; cparams.type_v = g_cache_type; + cparams.n_rs_seq = g_n_rs_seq; return llama_init_from_model(model, cparams); } @@ -282,6 +285,39 @@ int main(int argc, char ** argv) { llama_adapter_lora_free(adapter); } + if (llama_model_is_hybrid(model)) { + // 4. regression for the 5090 crash (2026-10-07 04:09Z): a recurrent context with rollback + // slots (serving's, inherited by the training context) made the exact walk ASSERT inside + // the serving process. It must refuse by name, and the adapter stays untouched. + llama_adapter_lora * adapter = llama_adapter_lora_init(model, init.c_str()); + const std::vector before = adapter_params(adapter); + g_n_rs_seq = 4; + llama_context * ctx = make_ctx(params, model, WINDOW / 4); + g_n_rs_seq = 0; + float scale = 1.0f; + GGML_ASSERT(llama_set_adapters_lora(ctx, &adapter, 1, &scale) == 0); + llama_opt_params lopt{}; + lopt.param_filter = llama_opt_param_filter_all; + lopt.get_opt_pars = sgd_pars; + lopt.optimizer_type = GGML_OPT_OPTIMIZER_TYPE_SGD; + lopt.adapter = adapter; + lopt.walk_exact = true; + llama_opt_init(ctx, model, lopt); + std::vector> seqs = { std::vector(tokens.begin(), tokens.begin() + WINDOW + 1) }; + std::vector> loss = { std::vector(WINDOW + 1, 1) }; + ggml_opt_dataset_t dataset = common_opt_dataset_init_masked(WINDOW, seqs, loss, tokens[0]); + llama_opt_epoch(ctx, dataset, nullptr, nullptr, /*idata_split =*/ 1, nullptr, nullptr); + const std::string why = llama_opt_failure(ctx); + printf(" rollback slots: failed=%d, \"%s\"\n", (int) llama_opt_failed(ctx), why.c_str()); + if (!llama_opt_failed(ctx) || why.find("rollback") == std::string::npos || adapter_params(adapter) != before) { + fprintf(stderr, "FAILED: a recurrent context with rollback slots was not refused by name\n"); + ++failures; + } + ggml_opt_dataset_free(dataset); + llama_free(ctx); + llama_adapter_lora_free(adapter); + } + std::remove(init.c_str()); llama_model_free(model); llama_backend_free(); diff --git a/tools/server/server-train.cpp b/tools/server/server-train.cpp index 9328d433ff17..1ae42b409473 100644 --- a/tools/server/server-train.cpp +++ b/tools/server/server-train.cpp @@ -751,6 +751,11 @@ void server_trainer::run(json req, examples_data ex) { // The training context: the SAME model, its own graph; flash attention has no backward. llama_context_params cparams = common_context_params_to_llama(params_base); + // NO ROLLBACK SLOTS. Serving keeps recurrent-state snapshots for speculative decoding's + // rollback (n_rs_seq, from the MTP draft); a training context never rolls back, and the exact + // walk carries one recurrent state per sequence. Inherited, they crashed the 5090's serving + // process on Kimi's first exact run (2026-10-07 04:09Z: the delta-net invariant asserted). + cparams.n_rs_seq = 0; // The context holds the whole window; one training chunk is one batch is one ubatch, the // largest multiple of 256 that divides the window and is at most the caller's "chunk" (the // memory gate's S). Context she did not write is decoded; her replies train chunk by chunk, From 7c96fb938f1663bc25b12e0c9a21a9b32e53c84c Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 6 Oct 2026 23:39:00 -0500 Subject: [PATCH 2/2] tests: a hybrid exact walk's state restores leave no non-consecutive positions (Fable on #48) Counts the recurrent memory's 'non-consecutive token position' warnings through a log callback across a hybrid exact walk: 0 with the position restore, 6 without it (mutation-checked), the 5090 log's 'position 65640 after 66345'. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- tests/test-walk-exact.cpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test-walk-exact.cpp b/tests/test-walk-exact.cpp index 0487f90f44ab..f13fe46f2231 100644 --- a/tests/test-walk-exact.cpp +++ b/tests/test-walk-exact.cpp @@ -21,6 +21,7 @@ #include "../src/llama-adapter.h" #include +#include #include #include #include @@ -44,6 +45,19 @@ static ggml_type g_cache_type = GGML_TYPE_F32; // recurrent rollback slots in the context (serving keeps some for speculative decoding) static uint32_t g_n_rs_seq = 0; +// counts the recurrent memory's "non-consecutive token position" warnings: a state restore that +// leaves the cell's position at the window's end makes every next chunk read as non-consecutive +// (the 5090 log, 2026-10-07: "position 65640 after 66345"), passing everything else through +static int g_nonconsecutive = 0; +static void count_log(ggml_log_level level, const char * text, void * ud) { + if (text && strstr(text, "non-consecutive token position")) { + ++g_nonconsecutive; + } + GGML_UNUSED(level); + GGML_UNUSED(ud); + fputs(text, stderr); +} + static llama_context * make_ctx(const common_params & params, llama_model * model, uint32_t chunk) { auto cparams = common_context_params_to_llama(params); cparams.n_ctx = WINDOW; @@ -285,6 +299,22 @@ int main(int argc, char ** argv) { llama_adapter_lora_free(adapter); } + if (llama_model_is_hybrid(model)) { + // 5. what this catches: a state restore that does not restore the recurrent cell's + // position. The reverse pass restores each chunk's entry state; with the cell left at the + // window's end, every chunk after the first restore reads as non-consecutive. + std::vector all(WINDOW + 1, 1); + all[0] = 0; + g_nonconsecutive = 0; + llama_log_set(count_log, nullptr); + step_delta(params, model, init, tokens, all, WINDOW / 4, true); + llama_log_set(nullptr, nullptr); // the default logger again + printf(" state restore: %d non-consecutive position warnings in the reverse pass\n", g_nonconsecutive); + if (g_nonconsecutive != 0) { + fprintf(stderr, "FAILED: a restored recurrent state left its cell at another position\n"); + ++failures; + } + } if (llama_model_is_hybrid(model)) { // 4. regression for the 5090 crash (2026-10-07 04:09Z): a recurrent context with rollback // slots (serving's, inherited by the training context) made the exact walk ASSERT inside