From 280497fbf3d14defdd64ec73d055189ee78c967d Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 01:08:24 +0900 Subject: [PATCH 01/12] lib: a file's first case and expectation instantiate no formatting and no std::function Under GCC a file of one case and one expectation took 689 ms to compile: std::format's machinery for eq() and std::function for the case's body, each loaded from the standard library's module into every file. Numbers and truth values are now compared and shown by the library (std::cmp_* for integers of either signedness), a case's body is a reference to the callable (a case runs while it is declared), a case declared in a body keeps its name and attributes in the library, and a suite of a name and a body that captures nothing passes a function pointer. The same file takes 265 ms; most of the rest is GCC loading the interface's classes. mcpplibs.os ^0.4.7: a capture no longer fsyncs its file twice per case. --- engine/mcpp.toml | 2 +- lib/mcpp.toml | 2 +- lib/src/mtest.cpp | 134 +++++++++++++++++++++- lib/src/mtest.cppm | 277 ++++++++++++++++++++++++++++++++++----------- 4 files changed, 341 insertions(+), 74 deletions(-) diff --git a/engine/mcpp.toml b/engine/mcpp.toml index 3c80487..31ca104 100644 --- a/engine/mcpp.toml +++ b/engine/mcpp.toml @@ -30,7 +30,7 @@ main = "src/main.cpp" [dependencies] mcpplibs.mtest.manifest = { path = "../manifest" } # the static case manifest -mcpplibs.os = "^0.4.1" +mcpplibs.os = "^0.4.7" mcpplibs.cmdline = "0.0.2" nlohmann.json = "3.12.0" marzer.tomlplusplus = "3.4.0" diff --git a/lib/mcpp.toml b/lib/mcpp.toml index 2a939ec..0d7d832 100644 --- a/lib/mcpp.toml +++ b/lib/mcpp.toml @@ -63,5 +63,5 @@ test-session = ["mtest", "serve"] # A range, not one release: the library is in the test graph of every package # that uses it, mcpplibs.os among them, whose tests build it against the os under # test (mcpp 2026.11.9.1), and a package pins its own os. -mcpplibs.os = "^0.4.1" +mcpplibs.os = "^0.4.7" nlohmann.json = "3.12.0" diff --git a/lib/src/mtest.cpp b/lib/src/mtest.cpp index e27fba2..c82a219 100644 --- a/lib/src/mtest.cpp +++ b/lib/src/mtest.cpp @@ -695,6 +695,97 @@ void run_suites() { // ---- expectations ------------------------------------------------------------ +std::string detail::show_text(std::string_view text) { + std::string out; + out.reserve(text.size() + 2); + out += '"'; + out += text; + out += '"'; + return out; +} +std::string detail::show_signed(long long value) { + return std::to_string(value); +} +std::string detail::show_unsigned(unsigned long long value) { + return std::to_string(value); +} +std::string detail::show_floating(double value, bool single) { + return single ? std::format("{}", static_cast(value)) : std::format("{}", value); +} +void detail::describe(comparison& c, const char* op) { + c.expression = c.actual + " " + op + " " + c.expected; +} + +namespace { +detail::operand as_double(const detail::operand& o) { + using of = detail::operand::of; + detail::operand d = o; + if (o.kind == of::signed_integer || o.kind == of::boolean) + d.d = static_cast(o.s); + else if (o.kind == of::unsigned_integer) + d.d = static_cast(o.u); + return d; +} +// -1, 0 or 1: a below, equal to or above b, by value. +int order(const detail::operand& a, const detail::operand& b) { + using of = detail::operand::of; + const bool floating = a.kind == of::floating || a.kind == of::single || b.kind == of::floating + || b.kind == of::single; + if (floating) { + const double x = as_double(a).d, y = as_double(b).d; + return x < y ? -1 : (y < x ? 1 : 0); + } + const bool ua = a.kind == of::unsigned_integer, ub = b.kind == of::unsigned_integer; + if (ua && ub) + return a.u < b.u ? -1 : (b.u < a.u ? 1 : 0); + if (ua) + return std::cmp_less(a.u, b.s) ? -1 : (std::cmp_less(b.s, a.u) ? 1 : 0); + if (ub) + return std::cmp_less(a.s, b.u) ? -1 : (std::cmp_less(b.u, a.s) ? 1 : 0); + return a.s < b.s ? -1 : (b.s < a.s ? 1 : 0); +} +std::string shown(const detail::operand& o) { + using of = detail::operand::of; + switch (o.kind) { + case of::boolean: return o.s ? "true" : "false"; + case of::signed_integer: return detail::show_signed(o.s); + case of::unsigned_integer: return detail::show_unsigned(o.u); + case of::single: return detail::show_floating(o.d, true); + case of::floating: return detail::show_floating(o.d, false); + } + return {}; +} +} // namespace + +// A comparison of two numbers or truth values. NaN is unordered: every relation but `!=` fails. +comparison detail::relate(const operand& a, relation r, const operand& b) { + const bool nan = std::isnan(as_double(a).d) || std::isnan(as_double(b).d); + const int o = nan ? 2 : order(a, b); + static constexpr const char* ops[] = {"==", "!=", "<", "<=", ">", ">="}; + comparison c; + switch (r) { + case relation::eq: c.held = o == 0; break; + case relation::ne: c.held = o != 0; break; + case relation::lt: c.held = o == -1; break; + case relation::le: c.held = o == -1 || o == 0; break; + case relation::gt: c.held = o == 1; break; + case relation::ge: c.held = o == 1 || o == 0; break; + } + if (!c.held) { + c.actual = shown(a); + c.expected = shown(b); + describe(c, ops[static_cast(r)]); + } + return c; +} + +comparison near(double a, double b, double tolerance) { + auto c = detail::compare(std::abs(a - b) <= tolerance, "~", a, b); + if (!c.held) + c.expression += std::format(" (tolerance {})", tolerance); + return c; +} + comparison contains(std::string_view text, std::string_view part) { comparison c; c.held = text.find(part) != std::string_view::npos; @@ -745,12 +836,17 @@ void fail(std::string message, std::source_location where) { // ---- cases -------------------------------------------------------------------- +struct test::state { + std::string name; + attributes attrs; +}; + void test::operator=(body b) { auto& s = the(); std::unique_lock g(s.lock); write_program(); - const std::string id = full_id(name_); - attributes attrs = within(inherited(), attrs_); + const std::string id = full_id(state_->name); + attributes attrs = within(inherited(), state_->attrs); attrs.skip = skip_reason(attrs); s.seen.insert(id); s.clauses.insert(attrs.clauses.begin(), attrs.clauses.end()); @@ -758,7 +854,7 @@ void test::operator=(body b) { if (s.listing) { event ev = {{"e", "case"}, {"id", id}, - {"name", name_}, + {"name", state_->name}, {"file", file}, {"line", b.where.line()}}; // A kind no attribute states is the runner's to give (the program's, by its directory). @@ -982,11 +1078,39 @@ void test::operator=(body b) { begin_capture(s.stack.back()); } -void Suite::add(std::string name, std::source_location where, attributes attrs, - std::function f) { +void detail::add_suite(std::string name, std::source_location where, attributes attrs, + std::function f) { the().suites.push_back({std::move(name), where, std::move(attrs), std::move(f)}); } +std::string detail::name_of(const char* data, std::size_t size) { + return size == static_cast(-1) ? std::string(data) : std::string(data, size); +} + +void detail::add_plain(const char* name, std::size_t size, std::source_location where, int k, + void (*f)()) { + attributes attrs; + if (k >= 0) + attrs.kind = static_cast(k); + add_suite(name_of(name, size), where, std::move(attrs), std::function(f)); +} + +// ---- a case declared in a body ------------------------------------------------------ + +test::test(std::string_view name) : state_(new state{std::string(name), {}}) {} +test::test(std::string_view name, attributes attrs) : state_(new state{std::string(name), std::move(attrs)}) {} +test::test(const char* name, std::size_t size) : state_(new state{std::string(name, size), {}}) {} +test::test(test&& o) noexcept : state_(std::exchange(o.state_, nullptr)) {} +test::~test() { + delete state_; +} +attributes& test::attrs() { + return state_->attrs; +} +std::string_view test::name() const { + return state_->name; +} + namespace { // MTEST_SERVE: this program serves one of its run fixtures for a runner (P11.3) and runs no case. std::optional serve_if_asked() { diff --git a/lib/src/mtest.cppm b/lib/src/mtest.cppm index d31e4ed..534e107 100644 --- a/lib/src/mtest.cppm +++ b/lib/src/mtest.cppm @@ -283,63 +283,125 @@ struct comparison { }; namespace detail { +// THE COMMON KINDS ARE SHOWN AND COMPARED BY THE LIBRARY ITSELF (0.11.0). A file's first use of +// std::format makes GCC load and instantiate its machinery, 0.5 s of a file of one case; the +// library's own functions are compiled once, in the library. +std::string show_text(std::string_view text); +std::string show_signed(long long value); +std::string show_unsigned(unsigned long long value); +std::string show_floating(double value, bool single); +template +inline constexpr bool is_character = std::is_same_v || std::is_same_v + || std::is_same_v || std::is_same_v || std::is_same_v + || std::is_same_v || std::is_same_v; template std::string show(const T& value) { - if constexpr (std::is_same_v, bool>) + using U = std::remove_cvref_t; + if constexpr (std::is_same_v) return value ? "true" : "false"; + else if constexpr (std::is_same_v || std::is_same_v) + return show_floating(value, std::is_same_v); + else if constexpr (std::is_integral_v && !is_character && std::is_signed_v) + return show_signed(value); + else if constexpr (std::is_integral_v && !is_character) + return show_unsigned(value); else if constexpr (std::is_convertible_v) - return std::format("\"{}\"", std::string_view(value)); + return show_text(std::string_view(value)); else if constexpr (std::formattable) return std::format("{}", value); else if constexpr (requires { std::to_underlying(value); }) - return std::format("{}", std::to_underlying(value)); + return show_signed(static_cast(std::to_underlying(value))); else - return std::format("<{} bytes>", sizeof(T)); + return "<" + show_unsigned(sizeof(T)) + " bytes>"; } +// The text of a failed comparison: "a op b". +void describe(comparison& c, const char* op); template -comparison compare(bool held, std::string_view op, const A& a, const B& b) { +comparison compare(bool held, const char* op, const A& a, const B& b) { comparison c; c.held = held; if (!held) { c.actual = show(a); c.expected = show(b); - c.expression = std::format("{} {} {}", c.actual, op, c.expected); + describe(c, op); } return c; } + +// A number or a truth value, compared in the library (0.11.0): integers by their values whatever +// their signedness (std::cmp_equal), a floating value with another number as double. +struct operand { + enum class of : unsigned char { signed_integer, unsigned_integer, floating, single, boolean } kind; + long long s = 0; + unsigned long long u = 0; + double d = 0; +}; +enum class relation : unsigned char { eq, ne, lt, le, gt, ge }; +comparison relate(const operand& a, relation r, const operand& b); +template +inline constexpr bool is_operand = std::is_same_v || std::is_same_v + || std::is_same_v || (std::is_integral_v && !is_character); +template +operand operand_of(const T& v) { + if constexpr (std::is_same_v) + return {operand::of::boolean, v ? 1 : 0, 0, 0}; + else if constexpr (std::is_same_v) + return {operand::of::single, 0, 0, v}; + else if constexpr (std::is_same_v) + return {operand::of::floating, 0, 0, v}; + else if constexpr (std::is_signed_v) + return {operand::of::signed_integer, static_cast(v), 0, 0}; + else + return {operand::of::unsigned_integer, 0, static_cast(v), 0}; +} +template +inline constexpr bool both_operands = is_operand> && is_operand>; } // namespace detail template comparison eq(const A& a, const B& b) { - return detail::compare(a == b, "==", a, b); + if constexpr (detail::both_operands) + return detail::relate(detail::operand_of(a), detail::relation::eq, detail::operand_of(b)); + else + return detail::compare(a == b, "==", a, b); } template comparison ne(const A& a, const B& b) { - return detail::compare(a != b, "!=", a, b); + if constexpr (detail::both_operands) + return detail::relate(detail::operand_of(a), detail::relation::ne, detail::operand_of(b)); + else + return detail::compare(a != b, "!=", a, b); } template comparison lt(const A& a, const B& b) { - return detail::compare(a < b, "<", a, b); + if constexpr (detail::both_operands) + return detail::relate(detail::operand_of(a), detail::relation::lt, detail::operand_of(b)); + else + return detail::compare(a < b, "<", a, b); } template comparison le(const A& a, const B& b) { - return detail::compare(a <= b, "<=", a, b); + if constexpr (detail::both_operands) + return detail::relate(detail::operand_of(a), detail::relation::le, detail::operand_of(b)); + else + return detail::compare(a <= b, "<=", a, b); } template comparison gt(const A& a, const B& b) { - return detail::compare(a > b, ">", a, b); + if constexpr (detail::both_operands) + return detail::relate(detail::operand_of(a), detail::relation::gt, detail::operand_of(b)); + else + return detail::compare(a > b, ">", a, b); } template comparison ge(const A& a, const B& b) { - return detail::compare(a >= b, ">=", a, b); + if constexpr (detail::both_operands) + return detail::relate(detail::operand_of(a), detail::relation::ge, detail::operand_of(b)); + else + return detail::compare(a >= b, ">=", a, b); } // |a - b| <= tolerance. -inline comparison near(double a, double b, double tolerance) { - auto c = detail::compare(std::abs(a - b) <= tolerance, "~", a, b); - if (!c.held) - c.expression += std::format(" (tolerance {})", tolerance); - return c; -} +comparison near(double a, double b, double tolerance); // `text` contains `part`. comparison contains(std::string_view text, std::string_view part); // `body` throws an exception of type E (any exception for E = void). @@ -427,15 +489,22 @@ expectation expect(fatal_condition value, // ---- cases ------------------------------------------------------------------ -// A case's body, with the place it was written. +// A case's body, with the place it was written. The callable is the one the declaration names; a +// case runs while it is declared, so the body refers to it and does not own it (0.11.0: an owning +// std::function made every file instantiate it, a quarter of a second under GCC). struct body { - std::function run; + void (*call)(void*) = nullptr; + void* object = nullptr; std::source_location where; // A body returns nothing: a case ends at its end, at `return;`, at skip() or at fail(). template requires std::invocable && std::is_void_v> - body(F f, std::source_location w = std::source_location::current()) - : run(std::move(f)), where(w) {} + body(F&& f, std::source_location w = std::source_location::current()) + : call([](void* o) { (*static_cast*>(o))(); }), + object(const_cast(static_cast(std::addressof(f)))), where(w) {} + void run() const { + call(object); + } }; // A case of several values: "x"_test = values{1, 2, 3} | [](int v) { ... }; @@ -463,13 +532,20 @@ parameterized operator|(values v, F f) { return {std::move(v.items), std::move(f), v.where}; } +// A case declared inside another's body, by `"name"_test = [&] { ... };`. Its name and attributes +// are the library's (0.11.0): a file that declares a case makes no std::string of its own. class test { public: - explicit test(std::string_view name) : name_(name) {} - test(std::string_view name, attributes attrs) : name_(name), attrs_(std::move(attrs)) {} + explicit test(std::string_view name); + test(std::string_view name, attributes attrs); + test(const char* name, std::size_t size); + test(test&&) noexcept; + test(const test&) = delete; + test& operator=(const test&) = delete; + ~test(); template test& operator[](A a) { - apply(attrs_, std::move(a)); + apply(attrs(), std::move(a)); return *this; } @@ -485,32 +561,87 @@ public: else return std::to_string(i); }(); - test one(std::format("{}/{}", name_, label)); - one.attrs_ = attrs_; + test one(std::string(name()) + "/" + label, attrs()); auto& f = p.f; - one.operator=(body([&f, &item] { std::invoke(f, item); }, p.where)); + auto run = [&f, &item] { std::invoke(f, item); }; + one.operator=(body(run, p.where)); } } private: - std::string name_; - attributes attrs_; + struct state; + attributes& attrs(); + std::string_view name() const; + state* state_ = nullptr; }; inline test operator""_test(const char* name, std::size_t size) { - return test(std::string_view(name, size)); + return test(name, size); } // A name and the place it is written: the first argument of a suite, converted where the suite is -// declared, so that the suite's case is reported at its declaration. +// declared, so that the suite's case is reported at its declaration. A literal is kept as its +// characters, without std::string_view (0.11.0), so that a file declaring a suite loads no more of +// the standard library than the suite needs. struct name_here { - std::string_view name; + const char* data = nullptr; + std::size_t size = 0; // all ones: up to the terminating NUL std::source_location where; template - requires std::convertible_to - name_here(const S& n, std::source_location w = std::source_location::current()) - : name(n), where(w) {} + requires std::is_array_v || std::convertible_to + name_here(const S& n, std::source_location w = std::source_location::current()) : where(w) { + if constexpr (std::is_array_v) { + data = n; + size = static_cast(-1); + } else { + const std::string_view v(n); + data = v.data(); + size = v.size(); + } + } }; +namespace detail { +std::string name_of(const char* data, std::size_t size); +void add_suite(std::string name, std::source_location where, attributes attrs, std::function f); +// A name (NUL-terminated when `size` is all ones), its place, its kind (-1: none) and its body. +void add_plain(const char* name, std::size_t size, std::source_location where, int k, void (*f)()); +template +void declare_suite(name_here name, int k, A&&... args) { + attributes attrs; + if (k >= 0) + attrs.kind = static_cast(k); + constexpr std::size_t n = sizeof...(A); + auto all = std::forward_as_tuple(std::forward(args)...); + using body_type = std::remove_cvref_t>>; + static_assert(std::invocable, "a suite ends with its body: [] { ... }"); + static_assert(std::is_void_v>, + "a body returns nothing: a case ends at its end, at `return;`, at skip() or " + "at fail()"); + [&](std::index_sequence) { + (apply(attrs, std::remove_cvref_t>>(std::get(all))), + ...); + }(std::make_index_sequence{}); + add_suite(name_of(name.data, name.size), name.where, std::move(attrs), + std::function(std::get(all))); +} +template +void declare_plain(const S& name, int k, void (*f)(), std::source_location where) { + if constexpr (std::is_array_v) { + add_plain(name, static_cast(-1), where, k, f); + } else { + const std::string_view n(name); + add_plain(n.data(), n.size(), where, k, f); + } +} +template +void declare_unnamed(F f, std::source_location where) { + static_assert(std::is_void_v>, + "a body returns nothing: a case ends at its end, at `return;`, at skip() or " + "at fail()"); + add_suite({}, where, attributes{}, std::function(std::move(f))); +} +} // namespace detail + // A case declared by an object at namespace scope: // // UTest parsing{"parsing", [] { expect(eq(parse(""), 0)); }}; @@ -532,70 +663,82 @@ struct name_here { // kind each. A suite without a name is no case: its body runs when the program does, and declares // cases of its own scope: // Suite all = [] { "a"_test = [] { ... }; }; +namespace detail { +// A body that captures nothing, which a suite at namespace scope always is: the library keeps a +// pointer to it, and a file that declares a suite of a name and a body instantiates nothing else. +template +inline constexpr bool plain_body = std::is_convertible_v; +template +inline constexpr bool one_plain_body = sizeof...(A) == 1 && (plain_body> && ...); +// The arguments of a declaration with attributes: what the plain form's own calls pass (a place) +// is none of them, so the declaration's overloads are set aside by their constraints before a +// name's conversion is examined, which would load std::string_view into the file. +template +inline constexpr bool declared_form = sizeof...(A) >= 1 && !one_plain_body + && (!std::is_same_v, std::source_location> && ...); +} // namespace detail + class Suite { public: template - requires(sizeof...(A) >= 1) + requires detail::declared_form Suite(name_here name, A&&... args) { - declare(name, attributes{}, std::forward(args)...); + detail::declare_suite(name, -1, std::forward(args)...); + } + template + requires detail::plain_body + Suite(const S& name, F f, std::source_location where = std::source_location::current()) { + detail::declare_plain(name, -1, f, where); } template requires std::invocable Suite(F f, std::source_location where = std::source_location::current()) { - static_assert(std::is_void_v>, - "a body returns nothing: a case ends at its end, at `return;`, at skip() or " - "at fail()"); - add({}, where, attributes{}, std::function(std::move(f))); + detail::declare_unnamed(std::move(f), where); } protected: template + requires detail::declared_form Suite(mtest::kind k, name_here name, A&&... args) { - attributes base; - base.kind = k; - declare(name, std::move(base), std::forward(args)...); + detail::declare_suite(name, static_cast(k), std::forward(args)...); } -private: - template - void declare(name_here name, attributes attrs, A&&... args) { - constexpr std::size_t n = sizeof...(A); - auto all = std::forward_as_tuple(std::forward(args)...); - using body_type = std::remove_cvref_t>>; - static_assert(std::invocable, "a suite ends with its body: [] { ... }"); - static_assert(std::is_void_v>, - "a body returns nothing: a case ends at its end, at `return;`, at skip() or " - "at fail()"); - [&](std::index_sequence) { - (apply(attrs, std::remove_cvref_t>>( - std::get(all))), - ...); - }(std::make_index_sequence{}); - add(std::string(name.name), name.where, std::move(attrs), - std::function(std::get(all))); + template + Suite(mtest::kind k, const S& name, void (*f)(), std::source_location where) { + detail::declare_plain(name, static_cast(k), f, where); } - static void add(std::string name, std::source_location where, attributes attrs, - std::function f); }; // Unit tests: a part of a program on its own. class UTest : public Suite { public: template - requires(sizeof...(A) >= 1) + requires detail::declared_form UTest(name_here name, A&&... args) : Suite(kind::unit, name, std::forward(args)...) {} + template + requires detail::plain_body + UTest(const S& name, F f, std::source_location where = std::source_location::current()) + : Suite(kind::unit, name, static_cast(f), where) {} }; // Conformance to a specification (P6.4): cases that name the clauses they cover. class CTS : public Suite { public: template - requires(sizeof...(A) >= 1) + requires detail::declared_form CTS(name_here name, A&&... args) : Suite(kind::cts, name, std::forward(args)...) {} + template + requires detail::plain_body + CTS(const S& name, F f, std::source_location where = std::source_location::current()) + : Suite(kind::cts, name, static_cast(f), where) {} }; // End-to-end tests: programs under test, run as their users run them (mcpplibs.mtest.e2e). class E2E : public Suite { public: template - requires(sizeof...(A) >= 1) + requires detail::declared_form E2E(name_here name, A&&... args) : Suite(kind::e2e, name, std::forward(args)...) {} + template + requires detail::plain_body + E2E(const S& name, F f, std::source_location where = std::source_location::current()) + : Suite(kind::e2e, name, static_cast(f), where) {} }; // Measurements (mcpplibs.mtest.bench). A measurement means something only on a machine nothing // else uses, so a Bench holds `machine` (P7.1). From fe17f1b926b3e14ae2187433f80951809cbeb6cf Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 01:15:03 +0900 Subject: [PATCH 02/12] Steps and the surface a case used (P5.6); a case's end checks its programs' processes and diagnostic lines; golden redactions lib: step() and exercise() write the step and exercise events; e2e::run writes a step for each command, with its status and the processor time of the programs it ran. at_case_end() registers a check the running case makes before its verdict. The e2e module fails a case whose program left a process holding an output open, and whose programs printed a line matching a project's diagnostic pattern (e2e::diagnostics) that the case did not declare (e2e::expect_diagnostic), or did not print a declared one; under the report policy (MTEST_DIAGNOSTICS=report) the lines are printed and counted as diagnostics.undeclared. golden() applies redactions (paths, durations, sizes, digests) and an update prints what it changed. engine: a case's steps, exercised items and processor time (from its cpu.user and cpu.sys metrics) are read from the report and kept in cases.ndjson. conformance: P5.6. --- conformance/src/fixture.cpp | 5 + conformance/tests/protocol.cpp | 22 ++++ engine/src/exec.cpp | 17 +++ engine/src/model.cpp | 34 ++++++ engine/src/model.cppm | 15 +++ lib/src/e2e.cpp | 208 ++++++++++++++++++++++++++++++++- lib/src/e2e.cppm | 36 +++++- lib/src/mtest.cpp | 64 ++++++++++ lib/src/mtest.cppm | 22 ++++ lib/tests/companions.cpp | 41 +++++++ lib/tests/expectations.cpp | 17 +++ 11 files changed, 478 insertions(+), 3 deletions(-) diff --git a/conformance/src/fixture.cpp b/conformance/src/fixture.cpp index 3f6d1b0..05ca3bf 100644 --- a/conformance/src/fixture.cpp +++ b/conformance/src/fixture.cpp @@ -19,5 +19,10 @@ int main() { "holds"_test[resource{"serial:db"}][artifact{"file:data/x.txt"}] = [] {}; "placed"_test[only{architecture::wasm32}] = [] {}; "bounded"_test[at_most{"elapsed", 0.1}] = [] { metric("elapsed", 0.5, "s"); }; + "steps"_test = [] { + step({.name = "build", .began = std::chrono::steady_clock::now(), + .took = std::chrono::milliseconds{3}, .status = 0, .build = "b1"}); + exercise({"command build", "flag build --target"}); + }; return report(); } diff --git a/conformance/tests/protocol.cpp b/conformance/tests/protocol.cpp index f0a7514..f0fa4a1 100644 --- a/conformance/tests/protocol.cpp +++ b/conformance/tests/protocol.cpp @@ -295,6 +295,28 @@ int main() { for (const auto& e : of(r, "end")) expect(find(r, "begin", e.value("id", "")).has_value()) << "end without begin"; }; + "P5.6 a step and the surface a case used are events of the case"_test[kind::cts][clause{"P5.6"}] = [] { + const auto r = run("run"); + const auto steps = of(r, "step"); + const auto used = of(r, "exercise"); + if (steps.empty() && used.empty()) + skip("the implementation writes no steps (P5.6 is optional)"); + for (const auto& st : steps) { + expect(find(r, "begin", st.value("id", "")).has_value()) << "a step outside a case: " << st.dump(); + expect(!st.value("name", "").empty()) << st.dump(); + expect(st.contains("t") && st["t"].is_number() && st.contains("ms") && st["ms"].is_number()) + << st.dump(); + } + for (const auto& u : used) { + expect(find(r, "begin", u.value("id", "")).has_value()) << "an exercise outside a case"; + expect(u.contains("items") && u["items"].is_array()) << u.dump(); + } + if (auto st = find(r, "step", id("steps"))) { + expect(eq(st->value("name", ""), std::string("build"))); + expect(eq(st->value("build", ""), std::string("b1"))); + expect(eq(st->value("status", -1), 0)); + } + }; "P5.5 the exit status is zero exactly when nothing failed"_test[kind::cts][clause{"P5.5"}] = [] { expect(ne(run("run").status, 0)); diff --git a/engine/src/exec.cpp b/engine/src/exec.cpp index 09a0e60..e45d371 100644 --- a/engine/src/exec.cpp +++ b/engine/src/exec.cpp @@ -282,6 +282,13 @@ report_reading read_report(std::string_view text, std::string_view program) { if (x.is_number()) m.samples.push_back(x.get()); at(id).metrics.push_back(std::move(m)); + } else if (kind == "step") { + at(id).steps.push_back(step_from(e)); + } else if (kind == "exercise") { + auto& c = at(id); + for (auto& item : strings_of(e, "items")) + if (std::ranges::find(c.exercised, item) == c.exercised.end()) + c.exercised.push_back(std::move(item)); } else if (kind == "artifact") { at(id).artifacts.push_back(e.value("path", "")); } else if (kind == "fixture") { @@ -291,6 +298,16 @@ report_reading read_report(std::string_view text, std::string_view program) { c.status = e.value("status", "fail"); c.ms = e.value("ms", 0.0); c.reason = e.value("reason", ""); + std::ranges::sort(c.exercised); + double cpu = 0; + bool told = false; + for (const auto& m : c.metrics) + if (m.name == "cpu.user" || m.name == "cpu.sys") { + cpu += m.value; + told = true; + } + if (told) + c.cpu_ms = cpu; } else if (kind == "missing") { auto& c = at(id); c.status = "fail"; diff --git a/engine/src/model.cpp b/engine/src/model.cpp index c863564..42b5700 100644 --- a/engine/src/model.cpp +++ b/engine/src/model.cpp @@ -125,6 +125,23 @@ json to_json(const case_result& c) { j["attempts"] = c.attempts; if (c.started_ms >= 0) j["started_ms"] = c.started_ms; + if (c.cpu_ms >= 0) + j["cpu_ms"] = c.cpu_ms; + if (!c.steps.empty()) { + j["steps"] = json::array(); + for (const auto& st : c.steps) { + json one = {{"name", st.name}, {"t", st.t}, {"ms", st.ms}}; + if (st.cpu_ms) + one["cpu_ms"] = *st.cpu_ms; + if (st.status) + one["status"] = *st.status; + if (!st.build.empty()) + one["build"] = st.build; + j["steps"].push_back(std::move(one)); + } + } + if (!c.exercised.empty()) + j["exercised"] = c.exercised; return j; } @@ -305,6 +322,18 @@ case_info case_info_from(const json& j) { c.reason = str(j, "reason"); return c; } +step_info step_from(const json& j) { + step_info st; + st.name = str(j, "name"); + st.t = j.value("t", 0.0); + st.ms = j.value("ms", 0.0); + if (auto it = j.find("cpu_ms"); it != j.end() && it->is_number()) + st.cpu_ms = it->get(); + if (auto it = j.find("status"); it != j.end() && it->is_number_integer()) + st.status = it->get(); + st.build = str(j, "build"); + return st; +} case_result case_result_from(const json& j) { case_result c; c.program = str(j, "program"); @@ -343,6 +372,11 @@ case_result case_result_from(const json& j) { c.fixtures.push_back({str(f, "name"), f.value("ms", 0.0)}); c.diagnosis = str(j, "diagnosis"); c.bounds = bounds_from(j); + c.cpu_ms = j.value("cpu_ms", -1.0); + if (auto it = j.find("steps"); it != j.end() && it->is_array()) + for (const auto& st : *it) + c.steps.push_back(step_from(st)); + c.exercised = strings(j, "exercised"); c.started_ms = j.value("started_ms", -1.0); return c; } diff --git a/engine/src/model.cppm b/engine/src/model.cppm index 1318e16..e209717 100644 --- a/engine/src/model.cppm +++ b/engine/src/model.cppm @@ -103,6 +103,15 @@ struct metric_info { std::vector samples; }; +// A step of a case (P5.6, 0.11.0): a command an end-to-end case ran. +struct step_info { + std::string name; + double t = 0, ms = 0; + std::optional cpu_ms; + std::optional status; + std::string build; // the record of the build the step was, when it was one +}; + // One case's result. An L0 program is one case whose identifier is the program's name. struct case_result { std::string program, id, kind = "unit"; @@ -121,6 +130,11 @@ struct case_result { // When it began, in milliseconds after the run began (its program's start and its `begin`'s // `t`); negative when not known. double started_ms = -1; + // Its processor time: the `cpu.user` and `cpu.sys` metrics it reported (P6.6), else the time + // its program used while it ran alone in it; negative when not known (0.11.0). + double cpu_ms = -1; + std::vector steps; // P5.6 (0.11.0) + std::vector exercised; // the items of the product's surface it used, sorted }; // A thread of a process, as the engine saw it at a timeout (0.10.0). @@ -227,6 +241,7 @@ json to_json(const fixture_run& f); fixture_run fixture_run_from(const json& j); case_info case_info_from(const json& j); +step_info step_from(const json& j); case_result case_result_from(const json& j); timeout_info timeout_info_from(const json& j); program_result program_result_from(const json& j); diff --git a/lib/src/e2e.cpp b/lib/src/e2e.cpp index d550900..bac85bd 100644 --- a/lib/src/e2e.cpp +++ b/lib/src/e2e.cpp @@ -12,6 +12,132 @@ constexpr std::string_view protocol_variables[] = {"MTEST_PROTOCOL", "MTEST_MOD "MTEST_REPORT", "MTEST_SELECT", "MTEST_ISOLATION", "MTEST_TMPDIR", "MTEST_SEED"}; + +// WHAT THE PROGRAMS OF A CASE DID THAT ITS EXPECTATIONS DO NOT SEE (0.11.0), judged at the case's +// end: a process that outlived the program that started it, holding an output open, and the +// lines of their output that match a diagnostic pattern of the project. +struct watch { + std::vector outlived; // the commands whose process outlived them + std::vector lines; // diagnostic lines, as written + std::vector declared; // the patterns the case declared +}; +std::mutex watch_lock; +std::map watches; // by case +std::vector project_patterns; +std::optional policy; + +diagnostic_policy current_policy() { + if (policy) + return *policy; + return os::env::get("MTEST_DIAGNOSTICS").value_or("") == "report" ? diagnostic_policy::report + : diagnostic_policy::enforce; +} + +bool matches_any(const std::string& line, const std::vector& patterns) { + for (const auto& p : patterns) + if (std::regex_search(line, std::regex(p))) + return true; + return false; +} + +void judge(const std::string& id) { + watch w; + { + std::lock_guard g(watch_lock); + auto it = watches.find(id); + if (it == watches.end()) + return; + w = std::move(it->second); + watches.erase(it); + } + for (const auto& line : w.outlived) { + comparison c; + c.expression = "a process the program started outlived it, holding an output open"; + c.actual = line; + expect(c) << "the case's programs leave nothing running"; + } + std::vector undeclared, unseen; + for (const auto& line : w.lines) + if (!matches_any(line, w.declared)) + undeclared.push_back(line); + for (const auto& pattern : w.declared) { + bool seen = false; + for (const auto& line : w.lines) + seen = seen || std::regex_search(line, std::regex(pattern)); + if (!seen) + unseen.push_back(pattern); + } + if (current_policy() == diagnostic_policy::report) { + metric("diagnostics.undeclared", static_cast(undeclared.size()), "1"); + for (const auto& line : undeclared) + std::println(std::cerr, "mtest: {}: an undeclared diagnostic line: {}", id, line); + return; + } + for (const auto& line : undeclared) { + comparison c; + c.expression = "a diagnostic line the case did not declare (e2e::expect_diagnostic)"; + c.actual = line; + expect(c) << "every diagnostic line is declared"; + } + for (const auto& pattern : unseen) { + comparison c; + c.expression = "a diagnostic the case declared did not appear"; + c.expected = pattern; + expect(c) << "every declared diagnostic appears"; + } +} + +watch& watch_of_case(const std::string& id) { + at_case_end("mcpplibs.mtest.e2e", [id] { judge(id); }); + return watches[id]; +} + +void observe(const outcome& r) { + const auto id = current_case(); + if (id.empty()) + return; + std::vector found; + std::vector patterns; + { + std::lock_guard g(watch_lock); + patterns = project_patterns; + } + if (!patterns.empty()) + for (const auto* text : {&r.out, &r.err}) + for (auto part : std::views::split(*text, '\n')) { + std::string line(part.begin(), part.end()); + if (!line.empty() && line.back() == '\r') + line.pop_back(); + if (!line.empty() && matches_any(line, patterns)) + found.push_back(std::move(line)); + } + if (found.empty() && !r.held_open) + return; + std::lock_guard g(watch_lock); + auto& w = watch_of_case(id); + if (r.held_open) + w.outlived.push_back(r.line); + for (auto& line : found) + w.lines.push_back(std::move(line)); +} +} // namespace + +void diagnostics(std::vector patterns) { + std::lock_guard g(watch_lock); + project_patterns = std::move(patterns); +} + +void expect_diagnostic(std::string pattern) { + const auto id = current_case(); + if (id.empty()) + return; + std::lock_guard g(watch_lock); + watch_of_case(id).declared.push_back(std::move(pattern)); +} + +void diagnostics_policy(diagnostic_policy p) { + std::lock_guard g(watch_lock); + policy = p; } outcome run(command c) { @@ -41,7 +167,18 @@ outcome run(command c) { limits.hard = c.timeout; limits.out_limit = c.out_limit; limits.err_limit = c.err_limit; + step_record st; + st.name = !c.step.empty() ? c.step : (r.line.size() > 200 ? r.line.substr(0, 200) + "..." : r.line); + st.began = std::chrono::steady_clock::now(); + const auto before = os::process::resources(os::process::of::children); auto done = os::process::run(std::move(o), limits, c.input); + st.took = std::chrono::steady_clock::now() - st.began; + if (const auto after = os::process::resources(os::process::of::children); before && after) + st.cpu_ms = std::chrono::duration((after->user + after->system) + - (before->user + before->system)).count(); + if (done) + st.status = done->status.shell_code(); + step(st); if (!done) { r.err = std::format("{} did not start: {}", c.argv.front(), done.error().message()); return r; @@ -55,6 +192,7 @@ outcome run(command c) { r.out = std::move(done->out); r.err = std::move(done->err); r.duration = done->duration; + observe(r); return r; } @@ -85,13 +223,80 @@ tree(std::initializer_list> files) return root; } +namespace { +// What a golden comparison compares: the text with each redaction applied, in order. +std::string redacted(std::string_view text, const std::vector& rules) { + std::string out(text); + for (const auto& r : rules) + out = std::regex_replace(out, std::regex(r.pattern), r.with); + return out; +} +std::vector lines_of(std::string_view text) { + std::vector out; + for (auto part : std::views::split(text, '\n')) + out.emplace_back(part.begin(), part.end()); + return out; +} +// What an update changed, by line: kept, changed in place, added and removed. +std::string update_summary(std::string_view before, std::string_view after) { + const auto a = lines_of(before), b = lines_of(after); + const std::size_t common = std::min(a.size(), b.size()); + std::size_t changed = 0; + for (std::size_t i = 0; i < common; ++i) + changed += a[i] != b[i]; + const std::size_t added = b.size() > a.size() ? b.size() - a.size() : 0; + const std::size_t removed = a.size() > b.size() ? a.size() - b.size() : 0; + return std::format("{} changed, {} added, {} removed", changed, added, removed); +} +} // namespace + +std::vector redactions::paths(const std::filesystem::path& root, std::string with) { + std::vector out; + for (const auto& spelling : {os::fs::utf8(root), os::fs::generic_utf8(root)}) { + std::string escaped; + for (char ch : spelling) { + if (std::string_view(R"(\^$.|?*+()[]{})").contains(ch)) + escaped += '\\'; + escaped += ch; + } + if (!escaped.empty()) + out.push_back({escaped, with}); + } + return out; +} +const std::vector& redactions::durations() { + static const std::vector r = {{R"(\b[0-9]+(\.[0-9]+)?\s?(ns|us|µs|ms|s|m)\b)", "