From fa25fc99c97526928453c2f5b435f105f096832e Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sun, 22 Nov 2026 05:23:44 +0900 Subject: [PATCH 1/2] 0.11.3: a ledger is judged by a set that runs every case A requirement is met by a passing case of the run's group, so a set that runs a part of the cases (`pr`, the cases a change affects) left unmet every requirement only the other cases meet, and its gate failed on every run whatever the change (mcpp's pull-request CI). `mtest gate` judges the ledgers for a set that runs every case (`full`, `release`, a declared set with `all`) and for a set it does not know; another set's gate judges the rest and states in a note (`notes`) that it did not judge them. The report writes the ledgers' table for any set. mtest's own tests hold under a runner that states its budget and serves run fixtures (mcpp 2026.11.21.1): the engine's runs under test state no outer budget, and the case that uses a fixture compares its endpoint with the one its runner stated. --- CHANGELOG.md | 14 ++++++++++++++ README.md | 2 +- README.zh-CN.md | 2 +- adapters/doctest/docs/writing-tests.md | 2 +- adapters/doctest/mcpp.toml | 4 ++-- adapters/doctest/src/protocol.cpp | 2 +- adapters/gtest/mcpp.toml | 2 +- adapters/gtest/src/protocol.cpp | 2 +- capi/mcpp.toml | 2 +- capi/src/mtest.cpp | 2 +- conformance/mcpp.toml | 2 +- docs/commands.md | 2 +- docs/config.md | 3 ++- docs/sets.md | 5 +++++ engine/mcpp.toml | 2 +- engine/src/analysis.cpp | 15 ++++++++++++++- engine/src/commands.cpp | 4 +++- engine/src/main.cpp | 4 ++-- engine/src/service.cpp | 2 +- engine/tests/engine.cpp | 8 ++++++++ lib/docs/writing-tests.md | 2 +- lib/mcpp.toml | 2 +- lib/src/mtest.cpp | 2 +- lib/tests/machine.cpp | 12 ++++++++++-- manifest/mcpp.toml | 2 +- 25 files changed, 76 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 620af76..89bf01f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.11.3] - 2026-11-22 + +- **A ledger is judged by a set that runs every case.** A requirement is met by a passing case of + the run's group, so a set that runs a part of the cases (`pr`, the cases a change affects) left + unmet every requirement only the other cases meet, and its gate failed on every run whatever the + change. `mtest gate` judges the ledgers for a set that runs every case (`full`, `release`, a + declared set with `all`) and for a set it does not know; for another set it judges the rest and + states in a note (`notes` in `--json`) that it did not judge them. `mtest report --sections + ledgers` writes their table for any set. Source-compatible with 0.11.2; the library changes only + the version it states. +- mtest's own tests hold under a runner that states its budget and serves run fixtures (mcpp + 2026.11.21.1): the engine's runs under test state no outer budget, and the case that uses a fixture + compares its endpoint with the one its runner stated. + ## [0.11.2] - 2026-11-21 - **What a scope does names no case.** A case that runs only because it declares a selected case diff --git a/README.md b/README.md index 4ecd5a4..034c5b7 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.11.2" +mcpplibs.mtest = "0.11.3" ``` A failure names its place and both sides (`expected 2, actual 3`). Under a runner the same program diff --git a/README.zh-CN.md b/README.zh-CN.md index 2a2623d..edd675a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -37,7 +37,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.11.2" +mcpplibs.mtest = "0.11.3" ``` 失败时给出位置与两侧的值(`expected 2, actual 3`)。在运行器之下,同一程序按协议报告;单独运行时每个用例输出一行。 diff --git a/adapters/doctest/docs/writing-tests.md b/adapters/doctest/docs/writing-tests.md index fa93867..321e1eb 100644 --- a/adapters/doctest/docs/writing-tests.md +++ b/adapters/doctest/docs/writing-tests.md @@ -5,7 +5,7 @@ package `mcpplibs.mtest.doctest` supplies doctest's implementation and the progr ```toml [dev-dependencies] -mcpplibs.mtest.doctest = "0.11.2" +mcpplibs.mtest.doctest = "0.11.3" ``` ```cpp diff --git a/adapters/doctest/mcpp.toml b/adapters/doctest/mcpp.toml index 188e52f..e07b66f 100644 --- a/adapters/doctest/mcpp.toml +++ b/adapters/doctest/mcpp.toml @@ -5,7 +5,7 @@ # neither (no `DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN`): # # [dev-dependencies] -# mcpplibs.mtest.doctest = "0.11.2" +# mcpplibs.mtest.doctest = "0.11.3" # # A case's identifier is `/` (P3.2), or `` outside a `TEST_SUITE`. doctest's # macros exist only in its header, so this package's one source includes it, and nlohmann/json's @@ -13,7 +13,7 @@ [package] namespace = "mcpplibs.mtest" name = "doctest" -version = "0.11.2" +version = "0.11.3" description = "doctest speaking the mtest protocol: doctest's implementation and a test main that lists, selects and reports its cases" license = "Apache-2.0" diff --git a/adapters/doctest/src/protocol.cpp b/adapters/doctest/src/protocol.cpp index cefd254..930de7e 100644 --- a/adapters/doctest/src/protocol.cpp +++ b/adapters/doctest/src/protocol.cpp @@ -162,7 +162,7 @@ int mcpplibs_mtest_doctest_main(int argc, char** argv) { {"protocol", 1}, {"level", 1}, {"framework", "doctest"}, - {"version", "0.11.2"}, + {"version", "0.11.3"}, {"isolation", {"process", "case"}}, {"guide", "https://github.com/mcpp-community/mtest/blob/main/adapters/doctest/docs/" "writing-tests.md"}}); diff --git a/adapters/gtest/mcpp.toml b/adapters/gtest/mcpp.toml index d37df57..e8d2385 100644 --- a/adapters/gtest/mcpp.toml +++ b/adapters/gtest/mcpp.toml @@ -13,7 +13,7 @@ [package] namespace = "mcpplibs.mtest" name = "gtest" -version = "0.11.2" +version = "0.11.3" description = "GoogleTest speaking the mtest protocol: a test main that lists, selects and reports GoogleTest's cases" license = "Apache-2.0" diff --git a/adapters/gtest/src/protocol.cpp b/adapters/gtest/src/protocol.cpp index cf84349..d79f5a3 100644 --- a/adapters/gtest/src/protocol.cpp +++ b/adapters/gtest/src/protocol.cpp @@ -128,7 +128,7 @@ int mcpplibs_mtest_gtest_main(int argc, char** argv) { {"protocol", 1}, {"level", 1}, {"framework", "googletest"}, - {"version", "0.11.2"}, + {"version", "0.11.3"}, {"isolation", {"process", "case"}}, {"guide", "https://github.com/mcpp-community/mtest/blob/main/adapters/gtest/docs/" "writing-tests.md"}}); diff --git a/capi/mcpp.toml b/capi/mcpp.toml index a48e81c..c1c5caf 100644 --- a/capi/mcpp.toml +++ b/capi/mcpp.toml @@ -4,7 +4,7 @@ [package] namespace = "mcpplibs.mtest" name = "capi" -version = "0.11.2" +version = "0.11.3" description = "mtest's C ABI: the protocol's events for any language with a C FFI" license = "Apache-2.0" diff --git a/capi/src/mtest.cpp b/capi/src/mtest.cpp index b75c003..9d397a2 100644 --- a/capi/src/mtest.cpp +++ b/capi/src/mtest.cpp @@ -120,7 +120,7 @@ void mtest_init(const char* framework, int level) { {"protocol", 1}, {"level", std::clamp(level, 1, 3)}, {"framework", text(framework)}, - {"version", "0.11.2"}, + {"version", "0.11.3"}, {"isolation", {"process", "case"}}}); } diff --git a/conformance/mcpp.toml b/conformance/mcpp.toml index b5a3615..9652d40 100644 --- a/conformance/mcpp.toml +++ b/conformance/mcpp.toml @@ -8,7 +8,7 @@ [package] namespace = "mcpplibs.mtest" name = "conformance" -version = "0.11.2" +version = "0.11.3" description = "The conformance suite of the mtest protocol, for every implementation of its program side" license = "Apache-2.0" diff --git a/docs/commands.md b/docs/commands.md index 922241d..237bd6f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -96,7 +96,7 @@ ended in the same way, with no record. | `query` | stored case results by `--status`, `--kind`, `--program`, `--case`, `--tag`, `--days`, `--runs` | | `analyze METRIC` | `hot`, `durations`, `path` (the critical path: the longest weighted path of the run's graph, spec P12.3, with what each step waited for), `failures` (grouped by message), `flaky`, `bench`, `trend` (each case's metrics over the last `--runs N` runs beside their bounds, `--case GLOB`), `cost` | | `graph [--format json\|dot\|mermaid\|gantt]` | sources → programs → cases → resources and artifacts; with `--run ID`, the run's graph of work -- builds, processes, run fixtures and what each waited for (worker, resource, build, fixture) -- the critical path marked, or its schedule as a gantt | -| `gate [--set S]` | a set's gate on a run: no failure, the set's budget kept, no case over `[budget] case_cpu` without a waiver, every ledger's requirement met or waived, no waiver that excuses nothing, bench verdicts that pass | +| `gate [--set S]` | a set's gate on a run: no failure, the set's budget kept, no case over `[budget] case_cpu` without a waiver, every ledger's requirement met or waived (by a set that runs every case; another set's gate notes that it did not judge them), no waiver that excuses nothing, bench verdicts that pass | | `coverage spread` | how widely the cases of the coverage map share files and functions: the median share, which decides whether selecting by a finer granularity selects fewer cases | A run is named by its identifier, `last`, or `last-N`. diff --git a/docs/config.md b/docs/config.md index f9fedb7..b2b4156 100644 --- a/docs/config.md +++ b/docs/config.md @@ -151,7 +151,8 @@ reason = "builds a platform's runtime for three configurations" # comment; `${NAME}` in the command is the variable's value). Each must be covered, over the run and # the runs of its group, by a passing case's clause (`evidence = "clauses"`, P6.4) or by an item of # the product's surface a passing case used (`evidence = "exercised"`, P5.6), or be excused by a -# waiver. `mtest gate` judges it and `mtest report --sections ledgers` writes its table. +# waiver. `mtest gate` judges it for a set that runs every case (a part leaves unmet what the other +# cases meet) and `mtest report --sections ledgers` writes its table. [ledger.surface] command = ["${MCPP}", "emit", "surface"] evidence = "exercised" diff --git a/docs/sets.md b/docs/sets.md index 6554b7f..7fa363a 100644 --- a/docs/sets.md +++ b/docs/sets.md @@ -14,3 +14,8 @@ under `[sets.]` ([config.md](config.md)). A declared set selects by `tags`, `kinds`, `programs` and `cases` (globs), excludes by `exclude_tags`, and `include`s other sets. A budget is reported when a run of the set exceeds it, naming its slowest cases; `--budget` instead keeps a run within it. + +A set's gate (`mtest gate --set S`) judges the ledgers only when the set runs every case (`full`, +`release`, or a declared set with `all = true`): a requirement is met by a passing case, and a set +that runs a part of the cases leaves unmet what only the others meet. Another set's gate judges its +failures and budgets and notes that it did not judge the ledgers. diff --git a/engine/mcpp.toml b/engine/mcpp.toml index 7c952e0..83b5dfd 100644 --- a/engine/mcpp.toml +++ b/engine/mcpp.toml @@ -10,7 +10,7 @@ [package] namespace = "mtest" name = "mtest" -version = "0.11.2" +version = "0.11.3" description = "The mtest engine: run, schedule, store, explain and analyse tests of any language" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/engine/src/analysis.cpp b/engine/src/analysis.cpp index 54ca24f..65f6b41 100644 --- a/engine/src/analysis.cpp +++ b/engine/src/analysis.cpp @@ -484,7 +484,19 @@ json gate(const config& c, const store& s, const run_record& r, std::string_view o.value("case", ""), o.value("cpu_ms", 0.0) / 1000, o.value("machine", ""), o.value("limit_ms", 0.0) / 1000)); } - for (const auto& l : ledger_table(c, s, r)) { + // A LEDGER IS JUDGED BY A SET THAT RUNS EVERY CASE (0.11.3). A requirement is met by a passing + // case of the run's group; a set that runs a part of the cases -- `pr`, the cases a change + // affects -- leaves unmet every requirement only the other cases meet, and its gate failed on + // every run, whatever the change (mcpp's pull-request CI, 2026-11-21). Such a gate says that it + // did not judge them; the report still writes their table. + json notes = json::array(); + const auto named = c.sets.find(std::string(set)); + const bool every_case = named == c.sets.end() || named->second.all; + if (!every_case && !c.ledgers.empty()) + notes.push_back(std::format("the ledgers are judged by a set that runs every case; {} runs " + "a part of them", + set)); + for (const auto& l : every_case ? ledger_table(c, s, r) : json::array()) { if (l.contains("error")) { reasons.push_back(l["error"].get()); continue; @@ -535,6 +547,7 @@ json gate(const config& c, const store& s, const run_record& r, std::string_view {"set", set}, {"pass", ok}, {"reasons", reasons}, + {"notes", notes}, {"bench", bench}, {"conclusion", ok ? std::format("the {} gate passes for run {}", set, r.id) : std::format("the {} gate fails: {}", set, diff --git a/engine/src/commands.cpp b/engine/src/commands.cpp index caf1439..ab1b795 100644 --- a/engine/src/commands.cpp +++ b/engine/src/commands.cpp @@ -1162,6 +1162,8 @@ answer cmd_gate(const std::filesystem::path& cwd, const json& p) { std::string text = g["conclusion"].get() + "\n"; for (const auto& x : g["reasons"]) text += " " + x.get() + "\n"; + for (const auto& x : g["notes"]) + text += " note: " + x.get() + "\n"; return {g["pass"].get() ? 0 : 1, g, text}; } @@ -2009,7 +2011,7 @@ json catalog() { plain({{"out", {{"type", "string"}}}, {"argv", {{"type", "array"}, {"items", {{"type", "string"}}}}}}), true); - return {{"schema", "mtest.catalog/1"}, {"version", "0.11.2"}, {"tools", tools}}; + return {{"schema", "mtest.catalog/1"}, {"version", "0.11.3"}, {"tools", tools}}; } } // namespace mcpplibs::mtest::engine diff --git a/engine/src/main.cpp b/engine/src/main.cpp index 8e62be1..d14522f 100644 --- a/engine/src/main.cpp +++ b/engine/src/main.cpp @@ -98,7 +98,7 @@ const json* tool_named(const json& tools, std::string_view name) { cmdline::App application(const json& tools) { auto app = cmdline::App("mtest") - .version("0.11.2") + .version("0.11.3") .description("run tests of any language, and ask about their runs") .option(cmdline::Option("json").global().help("print the answer's data")); for (const auto& t : tools) @@ -253,7 +253,7 @@ int main(int argc, char** argv) { const std::string name(parsed->subcommand_name()); const auto& sub = parsed->subcommand()->get(); if (name == "version") { - std::println("mtest 0.11.2"); + std::println("mtest 0.11.3"); return 0; } const auto cwd = std::filesystem::current_path(); diff --git a/engine/src/service.cpp b/engine/src/service.cpp index fd4c182..950bb5e 100644 --- a/engine/src/service.cpp +++ b/engine/src/service.cpp @@ -56,7 +56,7 @@ int serve(const std::filesystem::path& cwd) { {"result", {{"protocolVersion", params.value("protocolVersion", std::string("2025-06-18"))}, {"capabilities", {{"tools", {{"listChanged", false}}}}}, - {"serverInfo", {{"name", "mtest"}, {"version", "0.11.2"}}}, + {"serverInfo", {{"name", "mtest"}, {"version", "0.11.3"}}}, {"instructions", "Run tests and ask about runs: run, plan, explain, history, analyze, graph, " "gate, ci. A test session (test/plan, test/run, test/add, test/cancel, " diff --git a/engine/tests/engine.cpp b/engine/tests/engine.cpp index 348a66c..49ab5a1 100644 --- a/engine/tests/engine.cpp +++ b/engine/tests/engine.cpp @@ -37,8 +37,12 @@ project(std::initializer_list> fil e2e::outcome mtest(const std::filesystem::path& cwd, std::vector args, std::string input = {}) { args.insert(args.begin(), e2e::program("mtest")); + // The engine under test is a runner of its own. Inside the runner that runs these tests (mcpp + // 2026.11.21.1 states its budget to every program), it would take that budget, record none of + // its own, and the cases that state a run's budget would answer for the outer runner. return e2e::run({.argv = args, .cwd = cwd, + .env = {{"MTEST_BUDGET", std::nullopt}, {"MTEST_BUDGET_HELD", std::nullopt}}, .input = std::move(input), .timeout = std::chrono::seconds{120}}); } @@ -303,6 +307,10 @@ int main() { expect(!missing["pass"].get()) << missing.dump(); expect(contains(missing.dump(), "R2 has no passing case")) << missing.dump(); expect(!contains(missing.dump(), "R1 has no").held) << missing.dump(); + // A set that runs a part of the cases does not judge a ledger, and says so. + auto part = json_of(mtest(root, {"gate", "--set", "pr", "--json"})); + expect(part["pass"].get()) << part.dump(); + expect(contains(part.dump(), "judged by a set that runs every case")) << part.dump(); (void)mcpplibs::os::fs:: write(root / "mtest.toml", "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n[ledger.spec]\nfile = " diff --git a/lib/docs/writing-tests.md b/lib/docs/writing-tests.md index 86a6d59..2447082 100644 --- a/lib/docs/writing-tests.md +++ b/lib/docs/writing-tests.md @@ -53,7 +53,7 @@ E2E cli{"cli", resource{"serial:home"}, timeout{std::chrono::minutes{2}}, [] { / The entry enters the child before any case; the copy runs no case and reports nothing. The files of one directory are then one program (mcpp `[test] programs = "directory"`) however many of them start copies. -- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.11.2"`. +- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.11.3"`. - Identifiers are stable: a selection, a history and a shard name a case by them. ## Expectations diff --git a/lib/mcpp.toml b/lib/mcpp.toml index 4d9f5c2..3711731 100644 --- a/lib/mcpp.toml +++ b/lib/mcpp.toml @@ -17,7 +17,7 @@ [package] namespace = "mcpplibs" name = "mtest" -version = "0.11.2" +version = "0.11.3" description = "The C++ test library of mtest: cases, expectations, fixtures, bench, conformance and end-to-end helpers, with no macro and no header, speaking the mtest protocol" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/lib/src/mtest.cpp b/lib/src/mtest.cpp index 24ee615..5b9562b 100644 --- a/lib/src/mtest.cpp +++ b/lib/src/mtest.cpp @@ -345,7 +345,7 @@ void write_program() { {"protocol", 1}, {"level", 3}, {"framework", "mcpplibs.mtest"}, - {"version", "0.11.2"}, + {"version", "0.11.3"}, {"isolation", {"process", "case"}}, // P4.3: a selection's `!` and `?` lines are understood. {"selection", {"except", "optional"}}, diff --git a/lib/tests/machine.cpp b/lib/tests/machine.cpp index 3caded6..688ea69 100644 --- a/lib/tests/machine.cpp +++ b/lib/tests/machine.cpp @@ -33,10 +33,13 @@ run_fixture index{"index", [](serving& s) { }}; Component sandbox{"sandbox", {resource_pool{"port", 3}}}; +// The endpoint is the runner's: the test below states one to the copy it starts, and a runner that +// serves fixtures (mcpp 2026.11.21.1) gives the one this program served (`endpoint-of-index`). UTest user{"user", uses{"index"}, [] { - if (fixture_endpoint("index").empty()) + const auto given = mcpplibs::os::env::get("MTEST_FIXTURE_INDEX"); + if (!given || given->empty()) skip("no runner serves index here"); - expect(eq(fixture_endpoint("index"), std::string("from-runner"))); + expect(eq(fixture_endpoint("index"), *given)); }}; std::vector events_of(std::string_view mode, @@ -87,6 +90,11 @@ int main(int argc, char** argv) { const bool usage = std::ranges::any_of(events, [](const json& e) { return e.value("e", "") == "usage"; }); expect(usage) << "the program reports its own use at its end"; + // The case that uses the fixture read the endpoint stated to the copy. + const bool user_passed = std::ranges::any_of(events, [](const json& e) { + return e.value("e", "") == "end" && e.value("id", "") == "user" && e.value("status", "") == "pass"; + }); + expect(user_passed) << "the case that uses index did not pass with the endpoint `from-runner`"; }; "detailed: a case's largest resident set"_test = [] { if (mcpplibs::os::host() != mcpplibs::os::platform::linux || diff --git a/manifest/mcpp.toml b/manifest/mcpp.toml index 975fde1..a3db01b 100644 --- a/manifest/mcpp.toml +++ b/manifest/mcpp.toml @@ -8,7 +8,7 @@ [package] namespace = "mcpplibs.mtest" name = "manifest" -version = "0.11.2" +version = "0.11.3" description = "The static case manifest of mtest: the cases a source file declares, read without building it" license = "Apache-2.0" authors = ["mcpp-community"] From 0802ed47d3f14244b053612217b0124d1daea4e5 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sun, 22 Nov 2026 05:25:06 +0900 Subject: [PATCH 2/2] test: the fixture case's check is formatted --- lib/tests/machine.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/tests/machine.cpp b/lib/tests/machine.cpp index 688ea69..1c4f333 100644 --- a/lib/tests/machine.cpp +++ b/lib/tests/machine.cpp @@ -92,9 +92,11 @@ int main(int argc, char** argv) { expect(usage) << "the program reports its own use at its end"; // The case that uses the fixture read the endpoint stated to the copy. const bool user_passed = std::ranges::any_of(events, [](const json& e) { - return e.value("e", "") == "end" && e.value("id", "") == "user" && e.value("status", "") == "pass"; + return e.value("e", "") == "end" && e.value("id", "") == "user" && + e.value("status", "") == "pass"; }); - expect(user_passed) << "the case that uses index did not pass with the endpoint `from-runner`"; + expect(user_passed) + << "the case that uses index did not pass with the endpoint `from-runner`"; }; "detailed: a case's largest resident set"_test = [] { if (mcpplibs::os::host() != mcpplibs::os::platform::linux ||