From 722ad2055e63a2251e47da6c1311ab96391ac94d Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 10:24:38 +0900 Subject: [PATCH] 0.11.2: what a scope does names no case A case that runs only as the scope of a selected case (P3.4) is not reported unless a statement of its own fails. A step, exercise, artifact, metric or fixture event written while its body ran named it, and the runner held it as a case that did not end: a capability probe run by e2e::run in a suite's body failed a program whose cases all passed. These events now name no case. --- CHANGELOG.md | 9 +++++++++ README.md | 2 +- README.zh-CN.md | 2 +- adapters/doctest/docs/writing-tests.md | 2 +- adapters/doctest/mcpp.toml | 4 ++-- adapters/doctest/src/protocol.cpp | 2 +- adapters/gtest/mcpp.toml | 2 +- adapters/gtest/src/protocol.cpp | 2 +- capi/mcpp.toml | 2 +- capi/src/mtest.cpp | 2 +- conformance/mcpp.toml | 2 +- engine/mcpp.toml | 2 +- engine/src/commands.cpp | 2 +- engine/src/main.cpp | 4 ++-- engine/src/service.cpp | 2 +- lib/docs/writing-tests.md | 2 +- lib/mcpp.toml | 2 +- lib/src/mtest.cpp | 28 +++++++++++++++++++------- lib/src/mtest.cppm | 4 +++- lib/tests/suites.cpp | 7 +++++++ manifest/mcpp.toml | 2 +- spec/protocol.md | 1 + 22 files changed, 60 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f7ab8..620af76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.11.2] - 2026-11-21 + +- **What a scope does names no case.** A case that runs only because it declares a selected case + is that case's scope (P3.4) and is not reported unless a statement of its own fails. A `step`, + `exercise`, `artifact`, `metric` or `fixture` event written while its body ran named it, and the + runner, which never saw it begin, held it as a case that did not end: a capability probe run by + `e2e::run` in a suite's body failed the program whose cases all passed. These events now name no + case. Source-compatible with 0.11.1; the engine changes only the version it states. + ## [0.11.1] - 2026-11-21 - **A command is held to what its case has left.** `time_left()` states what the running case has diff --git a/README.md b/README.md index ca18f38..4ecd5a4 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.11.1" +mcpplibs.mtest = "0.11.2" ``` A failure names its place and both sides (`expected 2, actual 3`). Under a runner the same program diff --git a/README.zh-CN.md b/README.zh-CN.md index aeb8941..2a2623d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -37,7 +37,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.11.1" +mcpplibs.mtest = "0.11.2" ``` 失败时给出位置与两侧的值(`expected 2, actual 3`)。在运行器之下,同一程序按协议报告;单独运行时每个用例输出一行。 diff --git a/adapters/doctest/docs/writing-tests.md b/adapters/doctest/docs/writing-tests.md index a9083a2..fa93867 100644 --- a/adapters/doctest/docs/writing-tests.md +++ b/adapters/doctest/docs/writing-tests.md @@ -5,7 +5,7 @@ package `mcpplibs.mtest.doctest` supplies doctest's implementation and the progr ```toml [dev-dependencies] -mcpplibs.mtest.doctest = "0.11.1" +mcpplibs.mtest.doctest = "0.11.2" ``` ```cpp diff --git a/adapters/doctest/mcpp.toml b/adapters/doctest/mcpp.toml index 167024d..188e52f 100644 --- a/adapters/doctest/mcpp.toml +++ b/adapters/doctest/mcpp.toml @@ -5,7 +5,7 @@ # neither (no `DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN`): # # [dev-dependencies] -# mcpplibs.mtest.doctest = "0.11.1" +# mcpplibs.mtest.doctest = "0.11.2" # # A case's identifier is `/` (P3.2), or `` outside a `TEST_SUITE`. doctest's # macros exist only in its header, so this package's one source includes it, and nlohmann/json's @@ -13,7 +13,7 @@ [package] namespace = "mcpplibs.mtest" name = "doctest" -version = "0.11.1" +version = "0.11.2" description = "doctest speaking the mtest protocol: doctest's implementation and a test main that lists, selects and reports its cases" license = "Apache-2.0" diff --git a/adapters/doctest/src/protocol.cpp b/adapters/doctest/src/protocol.cpp index fa06d88..cefd254 100644 --- a/adapters/doctest/src/protocol.cpp +++ b/adapters/doctest/src/protocol.cpp @@ -162,7 +162,7 @@ int mcpplibs_mtest_doctest_main(int argc, char** argv) { {"protocol", 1}, {"level", 1}, {"framework", "doctest"}, - {"version", "0.11.1"}, + {"version", "0.11.2"}, {"isolation", {"process", "case"}}, {"guide", "https://github.com/mcpp-community/mtest/blob/main/adapters/doctest/docs/" "writing-tests.md"}}); diff --git a/adapters/gtest/mcpp.toml b/adapters/gtest/mcpp.toml index 43e08c3..d37df57 100644 --- a/adapters/gtest/mcpp.toml +++ b/adapters/gtest/mcpp.toml @@ -13,7 +13,7 @@ [package] namespace = "mcpplibs.mtest" name = "gtest" -version = "0.11.1" +version = "0.11.2" description = "GoogleTest speaking the mtest protocol: a test main that lists, selects and reports GoogleTest's cases" license = "Apache-2.0" diff --git a/adapters/gtest/src/protocol.cpp b/adapters/gtest/src/protocol.cpp index 0de61e6..cf84349 100644 --- a/adapters/gtest/src/protocol.cpp +++ b/adapters/gtest/src/protocol.cpp @@ -128,7 +128,7 @@ int mcpplibs_mtest_gtest_main(int argc, char** argv) { {"protocol", 1}, {"level", 1}, {"framework", "googletest"}, - {"version", "0.11.1"}, + {"version", "0.11.2"}, {"isolation", {"process", "case"}}, {"guide", "https://github.com/mcpp-community/mtest/blob/main/adapters/gtest/docs/" "writing-tests.md"}}); diff --git a/capi/mcpp.toml b/capi/mcpp.toml index f624bc3..a48e81c 100644 --- a/capi/mcpp.toml +++ b/capi/mcpp.toml @@ -4,7 +4,7 @@ [package] namespace = "mcpplibs.mtest" name = "capi" -version = "0.11.1" +version = "0.11.2" description = "mtest's C ABI: the protocol's events for any language with a C FFI" license = "Apache-2.0" diff --git a/capi/src/mtest.cpp b/capi/src/mtest.cpp index ab1037e..b75c003 100644 --- a/capi/src/mtest.cpp +++ b/capi/src/mtest.cpp @@ -120,7 +120,7 @@ void mtest_init(const char* framework, int level) { {"protocol", 1}, {"level", std::clamp(level, 1, 3)}, {"framework", text(framework)}, - {"version", "0.11.1"}, + {"version", "0.11.2"}, {"isolation", {"process", "case"}}}); } diff --git a/conformance/mcpp.toml b/conformance/mcpp.toml index e59cef6..b5a3615 100644 --- a/conformance/mcpp.toml +++ b/conformance/mcpp.toml @@ -8,7 +8,7 @@ [package] namespace = "mcpplibs.mtest" name = "conformance" -version = "0.11.1" +version = "0.11.2" description = "The conformance suite of the mtest protocol, for every implementation of its program side" license = "Apache-2.0" diff --git a/engine/mcpp.toml b/engine/mcpp.toml index 3268701..7c952e0 100644 --- a/engine/mcpp.toml +++ b/engine/mcpp.toml @@ -10,7 +10,7 @@ [package] namespace = "mtest" name = "mtest" -version = "0.11.1" +version = "0.11.2" description = "The mtest engine: run, schedule, store, explain and analyse tests of any language" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/engine/src/commands.cpp b/engine/src/commands.cpp index 51592d2..caf1439 100644 --- a/engine/src/commands.cpp +++ b/engine/src/commands.cpp @@ -2009,7 +2009,7 @@ json catalog() { plain({{"out", {{"type", "string"}}}, {"argv", {{"type", "array"}, {"items", {{"type", "string"}}}}}}), true); - return {{"schema", "mtest.catalog/1"}, {"version", "0.11.1"}, {"tools", tools}}; + return {{"schema", "mtest.catalog/1"}, {"version", "0.11.2"}, {"tools", tools}}; } } // namespace mcpplibs::mtest::engine diff --git a/engine/src/main.cpp b/engine/src/main.cpp index 066958d..8e62be1 100644 --- a/engine/src/main.cpp +++ b/engine/src/main.cpp @@ -98,7 +98,7 @@ const json* tool_named(const json& tools, std::string_view name) { cmdline::App application(const json& tools) { auto app = cmdline::App("mtest") - .version("0.11.1") + .version("0.11.2") .description("run tests of any language, and ask about their runs") .option(cmdline::Option("json").global().help("print the answer's data")); for (const auto& t : tools) @@ -253,7 +253,7 @@ int main(int argc, char** argv) { const std::string name(parsed->subcommand_name()); const auto& sub = parsed->subcommand()->get(); if (name == "version") { - std::println("mtest 0.11.1"); + std::println("mtest 0.11.2"); return 0; } const auto cwd = std::filesystem::current_path(); diff --git a/engine/src/service.cpp b/engine/src/service.cpp index e1d0d3a..fd4c182 100644 --- a/engine/src/service.cpp +++ b/engine/src/service.cpp @@ -56,7 +56,7 @@ int serve(const std::filesystem::path& cwd) { {"result", {{"protocolVersion", params.value("protocolVersion", std::string("2025-06-18"))}, {"capabilities", {{"tools", {{"listChanged", false}}}}}, - {"serverInfo", {{"name", "mtest"}, {"version", "0.11.1"}}}, + {"serverInfo", {{"name", "mtest"}, {"version", "0.11.2"}}}, {"instructions", "Run tests and ask about runs: run, plan, explain, history, analyze, graph, " "gate, ci. A test session (test/plan, test/run, test/add, test/cancel, " diff --git a/lib/docs/writing-tests.md b/lib/docs/writing-tests.md index 56c16fe..86a6d59 100644 --- a/lib/docs/writing-tests.md +++ b/lib/docs/writing-tests.md @@ -53,7 +53,7 @@ E2E cli{"cli", resource{"serial:home"}, timeout{std::chrono::minutes{2}}, [] { / The entry enters the child before any case; the copy runs no case and reports nothing. The files of one directory are then one program (mcpp `[test] programs = "directory"`) however many of them start copies. -- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.11.1"`. +- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.11.2"`. - Identifiers are stable: a selection, a history and a shard name a case by them. ## Expectations diff --git a/lib/mcpp.toml b/lib/mcpp.toml index d5691c4..4d9f5c2 100644 --- a/lib/mcpp.toml +++ b/lib/mcpp.toml @@ -17,7 +17,7 @@ [package] namespace = "mcpplibs" name = "mtest" -version = "0.11.1" +version = "0.11.2" description = "The C++ test library of mtest: cases, expectations, fixtures, bench, conformance and end-to-end helpers, with no macro and no header, speaking the mtest protocol" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/lib/src/mtest.cpp b/lib/src/mtest.cpp index 322fd22..24ee615 100644 --- a/lib/src/mtest.cpp +++ b/lib/src/mtest.cpp @@ -45,6 +45,9 @@ struct running_case { std::vector>> end_checks; // When it began: its `timeout` counts from here (time_left, 0.11.1). clock::time_point began = clock::now(); + // Reported: selected, or declared in a selected case. One that runs only because it declares + // a selected case is that case's scope; its begin is written only if it fails (0.11.2). + bool own = true; }; // The expectations that held, in the whole program: a case's share is the difference over its @@ -342,7 +345,7 @@ void write_program() { {"protocol", 1}, {"level", 3}, {"framework", "mcpplibs.mtest"}, - {"version", "0.11.1"}, + {"version", "0.11.2"}, {"isolation", {"process", "case"}}, // P4.3: a selection's `!` and `?` lines are understood. {"selection", {"except", "optional"}}, @@ -1052,6 +1055,7 @@ void test::operator=(body b) { end_capture(s.stack.back()); // the declaring case's output so far is its own s.stack.push_back({id, false, attrs}); s.stack.back().held_before = g_held.load(std::memory_order_relaxed); + s.stack.back().own = own; if (capturing) { s.stack.back().capturing = true; begin_capture(s.stack.back()); @@ -1379,6 +1383,16 @@ std::string current_case() { return s.stack.empty() || s.stack.back().scope ? std::string{} : s.stack.back().id; } +// The case an event of the running code states: the running case when it is reported, else none. +// A step, an artifact, a metric or a fixture of a case that runs as a scope would otherwise name a +// case the runner never saw begin, and the runner holds such a case as one that did not end. +static std::string event_case() { + auto& s = the(); + std::lock_guard g(s.lock); + return s.stack.empty() || s.stack.back().scope || !s.stack.back().own ? std::string{} + : s.stack.back().id; +} + std::optional time_left() { auto& s = the(); std::lock_guard g(s.lock); @@ -1395,7 +1409,7 @@ std::optional time_left() { void produced(const std::filesystem::path& file, std::string_view kind) { write_event({{"e", "artifact"}, - {"id", current_case()}, + {"id", event_case()}, {"path", os::fs::generic_utf8(file)}, {"kind", kind}}); } @@ -1442,7 +1456,7 @@ void at_case_end(std::string_view key, std::function check) { void step(const step_record& r) { const auto at = std::chrono::duration(r.began - the().began).count(); event ev = {{"e", "step"}, - {"id", current_case()}, + {"id", event_case()}, {"name", r.name}, {"t", at < 0 ? 0.0 : at}, {"ms", std::chrono::duration(r.took).count()}}; @@ -1458,7 +1472,7 @@ void step(const step_record& r) { void exercise(const std::vector& items) { if (items.empty()) return; - write_event({{"e", "exercise"}, {"id", current_case()}, {"items", items}}); + write_event({{"e", "exercise"}, {"id", event_case()}, {"items", items}}); } void metric(std::string_view name, double value, std::string_view unit, @@ -1469,8 +1483,8 @@ void metric(std::string_view name, double value, std::string_view unit, if (!s.stack.empty() && !s.stack.back().scope) s.stack.back().metrics[std::string(name)] = {value, std::string(unit)}; } - event ev = {{"e", "metric"}, {"id", current_case()}, {"name", name}, - {"value", value}, {"unit", unit}, {"better", better}}; + event ev = {{"e", "metric"}, {"id", event_case()}, {"name", name}, + {"value", value}, {"unit", unit}, {"better", better}}; if (!samples.empty()) ev["samples"] = std::vector(samples.begin(), samples.end()); write_event(ev); @@ -1760,7 +1774,7 @@ attributes within(const attributes& outer, const attributes& inner) { } void detail::made_shared(std::string_view name, double ms) { - write_event({{"e", "fixture"}, {"id", current_case()}, {"name", name}, {"ms", ms}}); + write_event({{"e", "fixture"}, {"id", event_case()}, {"name", name}, {"ms", ms}}); } std::vector self_command() { diff --git a/lib/src/mtest.cppm b/lib/src/mtest.cppm index 053d3f0..7e99158 100644 --- a/lib/src/mtest.cppm +++ b/lib/src/mtest.cppm @@ -857,7 +857,9 @@ void produced(const std::filesystem::path& file, std::string_view kind = "file") // A STEP OF THE RUNNING CASE (P5.6, 0.11.0): one command an end-to-end case ran, when it began, how // long it took, the processor time of the programs it ran, their exit status, and the record of // the build it was, when it was one. mcpplibs.mtest.e2e writes one for each command it runs; a kit -// that runs its commands another way writes its own. A step is a record and never a verdict. +// that runs its commands another way writes its own. A step is a record and never a verdict. A +// step, an exercise, an artifact, a metric or a fixture of a case that runs as the scope of a +// selected case (P3.4) names no case (0.11.2). struct step_record { std::string name; std::chrono::steady_clock::time_point began; diff --git a/lib/tests/suites.cpp b/lib/tests/suites.cpp index 14ef4fc..a5bf9e1 100644 --- a/lib/tests/suites.cpp +++ b/lib/tests/suites.cpp @@ -68,6 +68,9 @@ Suite kept{"kept", [] { Suite scoped{"scoped", [] { expect(!std::getenv("SCOPE_FAILS")) << "the scope's own statement"; + step({.name = "the scope's own command", + .began = std::chrono::steady_clock::now()}); + exercise({"scope.item"}); "inner"_test = [] {}; }}; @@ -242,6 +245,10 @@ int main(int argc, char** argv) { auto quiet = ended(events_of("run", {"scoped/inner"})); expect(eq(quiet["scoped/inner"], std::string("pass"))); expect(!quiet.contains("scoped")); + // What the scope's body does names no case: the runner saw none begin (0.11.2). + auto events = events_of("run", {"scoped/inner"}); + for (const auto& e : events) + expect(e.value("id", "") != "scoped") << "an event named the scope: " << e.dump(); auto loud = ended(events_of("run", {"scoped/inner"}, {}, {{"SCOPE_FAILS", "1"}})); expect(eq(loud["scoped"], std::string("fail"))); expect(eq(loud["scoped/inner"], std::string("pass"))); diff --git a/manifest/mcpp.toml b/manifest/mcpp.toml index 38e6a07..975fde1 100644 --- a/manifest/mcpp.toml +++ b/manifest/mcpp.toml @@ -8,7 +8,7 @@ [package] namespace = "mcpplibs.mtest" name = "manifest" -version = "0.11.1" +version = "0.11.2" description = "The static case manifest of mtest: the cases a source file declares, read without building it" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/spec/protocol.md b/spec/protocol.md index 57152d0..0a3bb7d 100644 --- a/spec/protocol.md +++ b/spec/protocol.md @@ -24,6 +24,7 @@ on the spec; this list says which of its clauses mtest's parts answer, and since | P3.3, places and capabilities | a `case` event states `only` and `except` (the platforms and architectures of `only{}`, `except{}`), `capabilities` (each `capability` the case requires, as the program answered it), and the `reason` of a case it lists skipped; the program skips a case outside its places or lacking a capability, whichever runner starts it | 0.8.0 | | B2: P4.3 | a selection's `!` lines leave out a case and the cases it declares, and its `?` lines select a case where it is declared and are not `missing` elsewhere; the `program` event states `selection: ["except", "optional"]` | 0.9.0 | | P5.6 | `step()` writes a `step` event and `exercise()` an `exercise` event of the running case; `e2e::run` writes a step for each command it runs | 0.11.0 | +| P5.6 within a scope | a `step`, `exercise`, `artifact`, `metric` or `fixture` event of a case that runs as the scope of a selected case (P3.4) names no case | 0.11.2 | | O5 within a case | `time_left()` states what a case has left of its `timeout` and its scopes'; `e2e::run` holds a command to it | 0.11.1 | | O12 `jitter` | `contention_point()` waits a random time up to `MTEST_JITTER`, the same for the same `MTEST_JITTER_SEED`; a case begins and a shared object is made at one | 0.11.0 | | O8 `output` | each case's output, kept while it runs under a runner and stated as `output` events; a case declared inside another keeps its own. Where the system cannot rearrange the program's streams (an ok image), no output is kept for a case, the spec's fallback | 0.6.0 |