diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a26843..77805e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,7 @@ jobs: MTEST_CTS_PROGRAM="$PWD/adapters/script/tests/fixture.py" mcpp test -p conformance MTEST_CTS_PROGRAM="$PWD/$(bin capi-fixture)" mcpp test -p conformance MTEST_CTS_PROGRAM="$PWD/$(bin gtest-fixture)" MTEST_CTS_ID_STYLE=gtest mcpp test -p conformance + MTEST_CTS_PROGRAM="$PWD/$(bin doctest-fixture)" MTEST_CTS_ID_STYLE=gtest mcpp test -p conformance - name: The engine runs mtest's own tests run: | set -euo pipefail diff --git a/AGENTS.md b/AGENTS.md index 843746e..9dc1161 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ protocol. This file states the core ideas and rules; the design and its record a | `manifest` | the static case manifest: the cases a source file declares, read from its text | nothing | | `engine` | the `mtest` command and its service | `manifest`, `mcpplibs.os`, `mcpplibs.cmdline`, JSON, TOML | | `conformance` | the protocol's conformance suite | `lib` | -| `adapters/*` | GoogleTest, Bash, Python | the framework adapted, JSON | +| `adapters/*` | GoogleTest, doctest, Bash, Python | the framework adapted, JSON | | `capi` | `include/mtest.h` for any language with a C FFI | JSON | ## Code @@ -42,7 +42,7 @@ protocol. This file states the core ideas and rules; the design and its record a - Format by [.clang-format](.clang-format) (the same as mcpp's); CI checks it with the pinned clang-format. - No macros: modules, `inline constexpr`, `if constexpr`, `std::source_location`. Headers appear - where a library is only a header: GoogleTest's in its adapter (with nlohmann/json's beside it), and + where a library is only a header: GoogleTest's and doctest's in their adapters (with nlohmann/json's beside them), and the C ABI's `include/mtest.h`. - Names are readable words; loops nest at most two deep (a helper otherwise). - Comments and documents are declarative and concise; they carry no local path, user name or diff --git a/CHANGELOG.md b/CHANGELOG.md index a6867ee..4b3230e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,58 @@ # Changelog +## [0.11.0] - 2026-11-21 + +What a test system needs to see the cost of the tests it runs (SPEC-017 v2.5; mcpp's test and CI +design of 2026-11-20). + +- **A file's first case and expectation instantiate no formatting.** Under GCC a test file of one + case and one expectation took 689 ms to compile: std::format's machinery for `eq()` and + std::function for the case's body were loaded into every file from the standard library's module. + Numbers and truth values are now compared and shown by the library (integers by their values, of + either signedness, as `std::cmp_equal` compares them; NaN is unordered), a case's body is a + reference to its callable (a case runs while it is declared), a case declared in a body keeps its + name and attributes in the library, and a suite of a name and a body that captures nothing passes + a function pointer. The same file takes 265 ms; most of the rest is GCC loading the interface's + classes. `comparison` keeps its fields. +- **The protocol costs a case 0.075 ms.** mcpplibs.os 0.4.7's capture no longer synchronises its + file twice per case: a program of 510 cases took 1.05 to 1.27 s under a runner that keeps each + case's output, and takes 38 ms. Both are now `Bench` cases with bounds (`lib/tests/costs.cpp`). +- **Steps and the surface a case used (P5.6).** `step()` and `exercise()` write the `step` and + `exercise` events; `e2e::run` writes a step for each command it runs, with its status and the + processor time of the programs it ran. The engine keeps a case's `steps`, `exercised`, `cpu_ms` + (its `cpu.user` and `cpu.sys` metrics) and `limit_ms`, and reads a host's `builds.ndjson`. +- **The end of a case checks what its programs did.** `at_case_end()` registers a check the running + case makes before its verdict. The e2e module fails a case whose program left a process holding an + output open, and whose programs printed a line matching a diagnostic pattern of the project + (`e2e::diagnostics`) that the case did not declare (`e2e::expect_diagnostic`), or did not print + one it declared; `MTEST_DIAGNOSTICS=report` prints and counts the lines instead. +- **Golden comparisons with redactions.** `e2e::golden(file, text, rules)` replaces paths, + durations, sizes and digests first; an update prints what it changed, by line. +- **One budget for the machine (O11).** A run makes a directory of one slot file per job; each worker + holds one and states it to its programs (`MTEST_BUDGET`, `MTEST_BUDGET_HELD`), so the parallel + work a program starts (a build's compilers) takes only the slots no worker holds. Inside another + runner's budget the engine uses that one. +- **Jitter (O12).** `mtest run --jitter MS [--jitter-seed N]`; `contention_point()` waits a random + time up to MS, the same for the same seed. A case begins and a shared object is made at one. +- **The machine's class.** `run.json`'s `machine` is the processor's model (mcpplibs.os 0.4.7), else + `/`, with `processors` and `memory`; through 0.10.0 it was the host's name. + Estimates prefer what was measured on the same class: hosted runners of one label differ by up to + 1.6 times. +- **`[budget]` and `[ledger.*]` are judged by the gate.** `case_cpu` bounds a case's processor time; + a ledger states requirements, the lines of a file or a command's output, each met by a passing + case's clause or used item of the surface, or excused by a waiver with a reason. A waiver that + excuses nothing fails the gate. +- **The coverage map records functions.** A run under `--coverage` keeps, per case, the functions + whose arc counters ran; `mtest coverage spread` gives the median share of cases that execute a + file and a function. +- **The Markdown report** gains the slowest steps, the builds a host recorded, the margin of the + cases near their limit, the budget and the ledgers; the summary names the machine. +- **doctest under the protocol (`mcpplibs.mtest.doctest`, L1).** doctest's implementation and the + test main are compiled once in the package: a test file includes the header and defines no + `DOCTEST_CONFIG_IMPLEMENT*`, which each file compiled again (1.6 s a file under GCC). It passes + the conformance suite. +- mcpplibs.os ^0.4.7. + ## [0.10.0] - 2026-11-19 - **A timeout records the program's processes, asks it to end, and then ends it.** Intermittent diff --git a/README.md b/README.md index b58f68c..c553e88 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Each part is used alone; together they share one data model. | C++ library | cases, expectations, fixtures; `bench`, `cts`, `e2e` modules | [lib/](lib/) — `mcpplibs.mtest` | | Engine | run, plan, explain, history, analyze, graph, gate, trace, bisect, ci | [engine/](engine/) — the `mtest` program | | Service | the engine's commands as JSON-RPC and MCP tools | `mtest serve` — [docs/service.md](docs/service.md) | -| Adapters | GoogleTest, Bash, Python | [adapters/](adapters/) — [docs/adapters.md](docs/adapters.md) | +| Adapters | GoogleTest, doctest, Bash, Python | [adapters/](adapters/) — [docs/adapters.md](docs/adapters.md) | | C ABI | the protocol for any language with a C FFI | [capi/](capi/) — `include/mtest.h` | | Conformance | one suite that checks every implementation of the protocol | [conformance/](conformance/) | @@ -38,7 +38,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" ``` A failure names its place and both sides (`expected 2, actual 3`). Under a runner the same program @@ -92,7 +92,7 @@ Every command takes `--json`, and every answer has a `conclusion`. See | L2 | metrics | bench verdicts, conformance clauses | | L3 | resources, artifacts | scheduling, affected selection | -The library, the C ABI and the script adapters are L3; GoogleTest is L1. +The library, the C ABI and the script adapters are L3; GoogleTest and doctest are L1. ## Build diff --git a/README.zh-CN.md b/README.zh-CN.md index 3aae0c3..97dcd20 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -15,7 +15,7 @@ | C++ 库 | 用例、期望、夹具;`bench`、`cts`、`e2e` 模块 | [lib/](lib/) — `mcpplibs.mtest` | | 引擎 | run、plan、explain、history、analyze、graph、gate、trace、bisect、ci | [engine/](engine/) — `mtest` 程序 | | 服务 | 以 JSON-RPC 与 MCP 工具提供引擎的命令 | `mtest serve` — [docs/service.md](docs/service.md) | -| 适配 | GoogleTest、Bash、Python | [adapters/](adapters/) — [docs/adapters.md](docs/adapters.md) | +| 适配 | GoogleTest、doctest、Bash、Python | [adapters/](adapters/) — [docs/adapters.md](docs/adapters.md) | | C ABI | 供任何有 C FFI 的语言使用的协议 | [capi/](capi/) — `include/mtest.h` | | 一致性 | 检验协议每一种实现的同一套用例 | [conformance/](conformance/) | @@ -37,7 +37,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" ``` 失败时给出位置与两侧的值(`expected 2, actual 3`)。在运行器之下,同一程序按协议报告;单独运行时每个用例输出一行。 @@ -83,7 +83,7 @@ mtest serve # 同样的命令,供 Agent(MCP)或 IDE | L2 | 指标 | bench 判定、一致性条款 | | L3 | 资源、产物 | 调度、受影响选择 | -库、C ABI 与脚本适配为 L3;GoogleTest 为 L1。 +库、C ABI 与脚本适配为 L3;GoogleTest 与 doctest 为 L1。 ## 构建 diff --git a/adapters/doctest/docs/writing-tests.md b/adapters/doctest/docs/writing-tests.md new file mode 100644 index 0000000..e807085 --- /dev/null +++ b/adapters/doctest/docs/writing-tests.md @@ -0,0 +1,25 @@ +# Writing tests with doctest under mtest + +A program's tests stay doctest's: `TEST_CASE`, `TEST_SUITE`, `CHECK`, `REQUIRE`, `SUBCASE`. The +package `mcpplibs.mtest.doctest` supplies doctest's implementation and the program's `main`, once: + +```toml +[dev-dependencies] +mcpplibs.mtest.doctest = "0.11.0" +``` + +```cpp +#include + +TEST_SUITE("parse") { + TEST_CASE("an empty line") { CHECK(parse_line("") == 0); } +} +``` + +- A test file includes the header and defines neither `DOCTEST_CONFIG_IMPLEMENT` nor + `DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN`: the implementation is compiled once, in the package, and not + in each file. +- A case's identifier is `/` (`parse/an empty line`), or `` outside a + `TEST_SUITE`; it is what `mcpp test --case` and `MTEST_SELECT` name. +- A case skipped by `doctest::skip()` is reported skipped, its `doctest::description` the reason. +- With no runner the program is doctest's own main and takes doctest's options. diff --git a/adapters/doctest/fixture/main.cpp b/adapters/doctest/fixture/main.cpp new file mode 100644 index 0000000..7c74be1 --- /dev/null +++ b/adapters/doctest/fixture/main.cpp @@ -0,0 +1,24 @@ +// The conformance fixture of the doctest adapter (spec/conformance.md), level 1: the cases doctest +// can state, named `/` (`conformance/passes`), with no tags, kinds or metrics. +// doctest's implementation and the main are the adapter's (src/protocol.cpp, src/main.cpp). +#include + +TEST_SUITE("conformance") { + TEST_CASE("passes") {} + TEST_CASE("fails") { + CHECK(1 == 2); + } + TEST_CASE("skips" * doctest::skip() * doctest::description("not here")) {} + TEST_CASE("measures") {} + TEST_CASE("covers") {} + TEST_CASE("holds") {} +} +TEST_SUITE("group") { + TEST_CASE("inner") {} +} + +int mcpplibs_mtest_doctest_main(int argc, char** argv); + +int main(int argc, char** argv) { + return mcpplibs_mtest_doctest_main(argc, argv); +} diff --git a/adapters/doctest/mcpp.toml b/adapters/doctest/mcpp.toml new file mode 100644 index 0000000..3acb26e --- /dev/null +++ b/adapters/doctest/mcpp.toml @@ -0,0 +1,46 @@ +# mcpplibs.mtest.doctest: doctest as an implementation of the mtest protocol, level 1 (spec/protocol.md): +# the cases listed, selected by identifier and reported with their failures' places, by a doctest +# reporter of the package's own. A program's tests stay doctest's (`TEST_CASE`, `CHECK`); the package +# supplies doctest's implementation and `main` once, so a test file includes the header and defines +# neither (no `DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN`): +# +# [dev-dependencies] +# mcpplibs.mtest.doctest = "0.11.0" +# +# A case's identifier is `/` (P3.2), or `` outside a `TEST_SUITE`. doctest's +# macros exist only in its header, so this package's one source includes it, and nlohmann/json's +# beside it, as the GoogleTest adapter does. +[package] +namespace = "mcpplibs.mtest" +name = "doctest" +version = "0.11.0" +description = "doctest speaking the mtest protocol: doctest's implementation and a test main that lists, selects and reports its cases" +license = "Apache-2.0" + +[language] +standard = "c++23" +modules = true +import_std = false + +[build] +sources = ["src/protocol.cpp"] + +[targets.doctest] +kind = "lib" + +[targets.main] +kind = "entry" +main = "src/main.cpp" +test-protocol = 1 +test-main = true +test-guide = "docs/writing-tests.md" + +# The adapter's own conformance fixture (spec/conformance.md), which the conformance suite runs +# (MTEST_CTS_PROGRAM, MTEST_CTS_ID_STYLE=gtest: `/`). +[targets.doctest-fixture] +kind = "bin" +main = "fixture/main.cpp" + +[dependencies] +compat.doctest = "2.4.12" +nlohmann.json = "3.12.0" diff --git a/adapters/doctest/src/main.cpp b/adapters/doctest/src/main.cpp new file mode 100644 index 0000000..aa9c744 --- /dev/null +++ b/adapters/doctest/src/main.cpp @@ -0,0 +1,7 @@ +// The `main` of a doctest program under the mtest protocol: the package's entry, which mcpp begins +// a test file that defines no `main` with (`test-main`). +int mcpplibs_mtest_doctest_main(int argc, char** argv); + +int main(int argc, char** argv) { + return mcpplibs_mtest_doctest_main(argc, argv); +} diff --git a/adapters/doctest/src/protocol.cpp b/adapters/doctest/src/protocol.cpp new file mode 100644 index 0000000..12999d3 --- /dev/null +++ b/adapters/doctest/src/protocol.cpp @@ -0,0 +1,212 @@ +// doctest under the mtest protocol (spec/protocol.md, level 1), and doctest's implementation. +// +// With no runner it is doctest's own main. Under one (MTEST_PROTOCOL and MTEST_REPORT), `list` mode +// writes the registered test cases as cases without running any (P3); a selection (MTEST_SELECT) +// runs exactly the named cases and an identifier no case has is `missing` (P4.1); a reporter writes +// each case's begin, failures and end (P5). A case doctest skips by its decorator is reported +// skipped, with its description as the reason. +#define DOCTEST_CONFIG_IMPLEMENT +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// P2.3: a runner that reads this program's file finds the protocol here. +extern "C" const char mcpplibs_mtest_doctest_marker[] = "mtest-protocol=1"; + +namespace { + +using event = nlohmann::ordered_json; + +std::string env(const char* name) { + const char* v = std::getenv(name); + return v ? v : ""; +} + +// The report (P5.1): one event per line, written whole. +struct report_file { + std::ofstream out; + void write(const event& e) { + if (!out) + return; + out << e.dump(-1, ' ', false, event::error_handler_t::replace) << "\n"; + out.flush(); + } +}; +report_file& report() { + static report_file r; + return r; +} + +// What a run selected; empty: every case. +std::set& selected() { + static std::set s; + return s; +} +std::map& counts() { + static std::map c; + return c; +} + +std::string id_of(const doctest::TestCaseData& t) { + const std::string suite = t.m_test_suite ? t.m_test_suite : ""; + return suite.empty() ? std::string(t.m_name) : suite + "/" + t.m_name; +} + +const auto began = std::chrono::steady_clock::now(); +long long ms_since(std::chrono::steady_clock::time_point at) { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - + at) + .count(); +} + +class mtest_reporter : public doctest::IReporter { +public: + explicit mtest_reporter(const doctest::ContextOptions&) {} + void report_query(const doctest::QueryData& q) override { + for (unsigned i = 0; i < q.num_data; ++i) { + const auto& t = *q.data[i]; + event e = {{"e", "case"}, {"id", id_of(t)}, + {"name", t.m_name}, {"file", t.m_file.c_str()}, + {"line", t.m_line}, {"kind", "unit"}}; + if (t.m_skip) { + e["expect"] = "skip"; + e["reason"] = t.m_description && *t.m_description ? t.m_description : "skipped"; + } + report().write(e); + } + } + void test_run_start() override {} + void test_run_end(const doctest::TestRunStats&) override {} + void test_case_start(const doctest::TestCaseData& t) override { + current_ = id_of(t); + started_ = std::chrono::steady_clock::now(); + report().write({{"e", "begin"}, {"id", current_}, {"t", ms_since(began)}}); + } + void test_case_reenter(const doctest::TestCaseData&) override {} + void test_case_end(const doctest::CurrentTestCaseStats& st) override { + const std::string status = st.testCaseSuccess ? "pass" : "fail"; + ++counts()[status]; + report().write( + {{"e", "end"}, {"id", current_}, {"status", status}, {"ms", ms_since(started_)}}); + } + void test_case_exception(const doctest::TestCaseException& e) override { + report().write({{"e", "failure"}, + {"id", current_}, + {"file", ""}, + {"line", 0}, + {"message", std::string(e.is_crash ? "crashed: " : "threw: ") + + e.error_string.c_str()}, + {"fatal", true}}); + } + void subcase_start(const doctest::SubcaseSignature&) override {} + void subcase_end() override {} + void log_assert(const doctest::AssertData& a) override { + if (!a.m_failed) + return; + std::string message = a.m_expr ? a.m_expr : ""; + if (a.m_threw) + message += std::string(" threw: ") + a.m_exception.c_str(); + else if (a.m_decomp.size()) + message += std::string(" (") + a.m_decomp.c_str() + ")"; + event e = {{"e", "failure"}, + {"id", current_}, + {"file", a.m_file ? a.m_file : ""}, + {"line", a.m_line}, + {"message", message}}; + if (a.m_at & doctest::assertType::is_require) + e["fatal"] = true; + report().write(e); + } + void log_message(const doctest::MessageData&) override {} + void test_case_skipped(const doctest::TestCaseData& t) override { + const auto id = id_of(t); + // A case a selection left out is not reported; one its decorator skipped is. + if (!selected().empty() && !selected().contains(id)) + return; + const std::string reason = + t.m_description && *t.m_description ? t.m_description : "skipped"; + ++counts()["skip"]; + report().write({{"e", "begin"}, {"id", id}, {"t", ms_since(began)}}); + report().write( + {{"e", "end"}, {"id", id}, {"status", "skip"}, {"ms", 0}, {"reason", reason}}); + } +private: + std::string current_; + std::chrono::steady_clock::time_point started_; +}; + +} // namespace + +REGISTER_REPORTER("mtest", 1, mtest_reporter); + +// The protocol's main of a doctest program; src/main.cpp (the package's entry) calls it. +extern "C++" int mcpplibs_mtest_doctest_main(int argc, char** argv); + +int mcpplibs_mtest_doctest_main(int argc, char** argv) { + doctest::Context context; + context.applyCommandLine(argc, argv); + const std::string path = env("MTEST_REPORT"); + if (env("MTEST_PROTOCOL").empty() || path.empty()) + return context.run(); + const char* volatile keep = mcpplibs_mtest_doctest_marker; + (void)keep; + report().out.open(path, std::ios::app | std::ios::binary); + report().write( + {{"e", "program"}, + {"protocol", 1}, + {"level", 1}, + {"framework", "doctest"}, + {"version", "0.11.0"}, + {"isolation", {"process", "case"}}, + {"guide", "https://github.com/mcpp-community/mtest/blob/main/adapters/doctest/docs/" + "writing-tests.md"}}); + // The list is the registry's: doctest's own listing leaves out a case its decorator skips. + if (env("MTEST_MODE") == "list") { + std::vector all; + for (const auto& t : doctest::detail::getRegisteredTests()) + all.push_back(&t); + doctest::QueryData q; + q.data = all.data(); + q.num_data = static_cast(all.size()); + mtest_reporter(doctest::ContextOptions{}).report_query(q); + report().write({{"e", "done"}, {"status", "pass"}, {"counts", event::object()}}); + return 0; + } + // doctest's options as its command line spells them: setOption does not set every option. + const char* options[] = {argc > 0 ? argv[0] : "test", "--reporters=mtest"}; + context.applyCommandLine(2, options); + bool missing = false; + if (const auto select = env("MTEST_SELECT"); !select.empty()) { + std::set known; + for (const auto& t : doctest::detail::getRegisteredTests()) + known.insert(id_of(t)); + std::ifstream in(select); + for (std::string line; std::getline(in, line);) { + if (!line.empty() && line.back() == '\r') + line.pop_back(); + if (line.empty()) + continue; + if (!known.contains(line)) { + report().write({{"e", "missing"}, {"id", line}}); + missing = true; + continue; + } + selected().insert(line); + } + // A case the selection leaves out is skipped: doctest runs no case marked so, and its + // filters match names by patterns, which an identifier with `*` or `,` would defeat. + for (const auto& t : doctest::detail::getRegisteredTests()) + if (!selected().contains(id_of(t))) + const_cast(t).m_skip = true; + } + const int status = context.run(); + const bool failed = status != 0 || missing || counts()["fail"] > 0; + report().write({{"e", "done"}, {"status", failed ? "fail" : "pass"}, {"counts", counts()}}); + return failed ? 1 : 0; +} diff --git a/adapters/gtest/mcpp.toml b/adapters/gtest/mcpp.toml index a41302b..d94e3d6 100644 --- a/adapters/gtest/mcpp.toml +++ b/adapters/gtest/mcpp.toml @@ -13,7 +13,7 @@ [package] namespace = "mcpplibs.mtest" name = "gtest" -version = "0.10.0" +version = "0.11.0" description = "GoogleTest speaking the mtest protocol: a test main that lists, selects and reports GoogleTest's cases" license = "Apache-2.0" diff --git a/adapters/gtest/src/protocol.cpp b/adapters/gtest/src/protocol.cpp index 3b7a57b..e47dd1d 100644 --- a/adapters/gtest/src/protocol.cpp +++ b/adapters/gtest/src/protocol.cpp @@ -128,7 +128,7 @@ int mcpplibs_mtest_gtest_main(int argc, char** argv) { {"protocol", 1}, {"level", 1}, {"framework", "googletest"}, - {"version", "0.10.0"}, + {"version", "0.11.0"}, {"isolation", {"process", "case"}}, {"guide", "https://github.com/mcpp-community/mtest/blob/main/adapters/gtest/docs/" "writing-tests.md"}}); diff --git a/capi/mcpp.toml b/capi/mcpp.toml index 611298e..9c4f801 100644 --- a/capi/mcpp.toml +++ b/capi/mcpp.toml @@ -4,7 +4,7 @@ [package] namespace = "mcpplibs.mtest" name = "capi" -version = "0.10.0" +version = "0.11.0" description = "mtest's C ABI: the protocol's events for any language with a C FFI" license = "Apache-2.0" diff --git a/capi/src/mtest.cpp b/capi/src/mtest.cpp index cdd7ef2..c30cd2e 100644 --- a/capi/src/mtest.cpp +++ b/capi/src/mtest.cpp @@ -120,7 +120,7 @@ void mtest_init(const char* framework, int level) { {"protocol", 1}, {"level", std::clamp(level, 1, 3)}, {"framework", text(framework)}, - {"version", "0.10.0"}, + {"version", "0.11.0"}, {"isolation", {"process", "case"}}}); } diff --git a/conformance/mcpp.toml b/conformance/mcpp.toml index 2916565..bdf0695 100644 --- a/conformance/mcpp.toml +++ b/conformance/mcpp.toml @@ -8,7 +8,7 @@ [package] namespace = "mcpplibs.mtest" name = "conformance" -version = "0.10.0" +version = "0.11.0" description = "The conformance suite of the mtest protocol, for every implementation of its program side" license = "Apache-2.0" diff --git a/conformance/src/fixture.cpp b/conformance/src/fixture.cpp index 3f6d1b0..6bbe69e 100644 --- a/conformance/src/fixture.cpp +++ b/conformance/src/fixture.cpp @@ -19,5 +19,13 @@ int main() { "holds"_test[resource{"serial:db"}][artifact{"file:data/x.txt"}] = [] {}; "placed"_test[only{architecture::wasm32}] = [] {}; "bounded"_test[at_most{"elapsed", 0.1}] = [] { metric("elapsed", 0.5, "s"); }; + "steps"_test = [] { + step({.name = "build", + .began = std::chrono::steady_clock::now(), + .took = std::chrono::milliseconds{3}, + .status = 0, + .build = "b1"}); + exercise({"command build", "flag build --target"}); + }; return report(); } diff --git a/conformance/tests/protocol.cpp b/conformance/tests/protocol.cpp index f0a7514..fd76f2d 100644 --- a/conformance/tests/protocol.cpp +++ b/conformance/tests/protocol.cpp @@ -295,6 +295,31 @@ int main() { for (const auto& e : of(r, "end")) expect(find(r, "begin", e.value("id", "")).has_value()) << "end without begin"; }; + "P5.6 a step and the surface a case used are events of the case"_test[kind::cts][clause{ + "P5.6"}] = [] { + const auto r = run("run"); + const auto steps = of(r, "step"); + const auto used = of(r, "exercise"); + if (steps.empty() && used.empty()) + skip("the implementation writes no steps (P5.6 is optional)"); + for (const auto& st : steps) { + expect(find(r, "begin", st.value("id", "")).has_value()) + << "a step outside a case: " << st.dump(); + expect(!st.value("name", "").empty()) << st.dump(); + expect(st.contains("t") && st["t"].is_number() && st.contains("ms") && + st["ms"].is_number()) + << st.dump(); + } + for (const auto& u : used) { + expect(find(r, "begin", u.value("id", "")).has_value()) << "an exercise outside a case"; + expect(u.contains("items") && u["items"].is_array()) << u.dump(); + } + if (auto st = find(r, "step", id("steps"))) { + expect(eq(st->value("name", ""), std::string("build"))); + expect(eq(st->value("build", ""), std::string("b1"))); + expect(eq(st->value("status", -1), 0)); + } + }; "P5.5 the exit status is zero exactly when nothing failed"_test[kind::cts][clause{"P5.5"}] = [] { expect(ne(run("run").status, 0)); diff --git a/docs/adapters.md b/docs/adapters.md index 9003a84..5edd8a6 100644 --- a/docs/adapters.md +++ b/docs/adapters.md @@ -8,6 +8,7 @@ conformance suite (`MTEST_CTS_PROGRAM`, a command: a program and its arguments, |---|---|---| | `mcpplibs.mtest` (C++) | 3 | `import mcpplibs.mtest;` — the library itself | | GoogleTest | 1 | `mcpplibs.mtest.gtest` as a dev-dependency, whose entry begins a test file (mcpp 2026.11.8.1+; earlier: `[test] main = { from = "mcpplibs.mtest.gtest/main" }` and `compat.gtest` as a dev-dependency too); identifiers are `/` | +| doctest | 1 | `mcpplibs.mtest.doctest` as a dev-dependency: doctest's implementation and the entry that begins a test file, compiled once; a test file includes `` and defines no `DOCTEST_CONFIG_IMPLEMENT*` (0.11.0); identifiers are `/`, or `` outside a suite | | Bash | 3 | `. mtest.sh`, then `mtest_case ID FUNCTION [--tag T ...]` and `mtest_report` | | Python | 3 | `import mtest_report as mt`, `@mt.case("id")`, `mt.report()` | | C ABI | 3 | `include/mtest.h` and `libmtest`, for any language with a C FFI | diff --git a/docs/commands.md b/docs/commands.md index 13231d8..922241d 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -36,7 +36,7 @@ nothing; a built program's own listing is the authority (`manifest/src/manifest. | Command | Does | |---|---| -| `run` | runs the selection; `--jobs N`, `--seed N`, `--fail-fast`, `--retries N`, `--isolate` (a process per case), `--timeout 5m`, `--grace 2s` (below), `--env K=V`, `--run-under CMD` (every start of a program, its listing included: a runner such as an emulator or a machine of the program's system), `--monitor LEVEL` (P10), `--group G` (runs reported together); `--junit`, `--html`, `--report-json FILE`, `--md FILE`; `--coverage` measures the coverage map. A program's cases are divided among workers that would otherwise idle (`[run] split`), and programs run together within the machine's memory (`[run] memory`) | +| `run` | runs the selection; `--jobs N`, `--seed N`, `--fail-fast`, `--retries N`, `--isolate` (a process per case), `--timeout 5m`, `--grace 2s` (below), `--env K=V`, `--run-under CMD` (every start of a program, its listing included: a runner such as an emulator or a machine of the program's system), `--monitor LEVEL` (P10), `--group G` (runs reported together), `--jitter MS [--jitter-seed N]` (each program waits a random time up to MS at its contention points, SPEC-017 O12: a search for an order defect, never a verdict); `--junit`, `--html`, `--report-json FILE`, `--md FILE`; `--coverage` measures the coverage map. A program's cases are divided among workers that would otherwise idle (`[run] split`), and programs run together within the machine's memory (`[run] memory`) | | `run --resume RUN` | the selection, less the cases RUN judged: their results are carried into the new run, so an interrupted long run (a bench over a matrix) ends as one whole run; a case RUN could not start runs again | | `rerun` | the cases that failed in the last run | | `smoke` | the `smoke` set | @@ -45,6 +45,13 @@ nothing; a built program's own listing is the authority (`manifest/src/manifest. | `bisect --case REF --good C [--bad C] [--build CMD]` | the first commit at which a case fails (git bisect, one case per step) | | `status`, `cancel` | runs in progress; stop one after the programs now running | +A run holds one budget for the machine (SPEC-017 O11): a directory of a slot file per job, each +worker holding one while it runs programs and stating it to them (`MTEST_BUDGET`, +`MTEST_BUDGET_HELD`), so the parallel work a program starts takes only the slots no worker holds. +Inside another runner's budget the engine uses that one. A run records the machine's class (the +processor's model, else `/`), and estimates prefer what was measured on the same +class. + A program runs by the protocol when it carries the marker or declares its level; otherwise its exit status is its one result. Cases tagged `flaky`, and quarantined ones, are retried alone up to `--retries`; one that then passes is `flaky`. A quarantined case's failure does not fail the run. @@ -83,13 +90,14 @@ ended in the same way, with no record. | Command | Answers | |---|---| | `explain [--case REF]` | where a case failed, why (both sides), its output, its recent history, whether it is new, and the command that repeats it | -| `report [--format text\|json\|junit\|html\|md]` | a stored run; `md`: a Markdown report of the sections `--sections` (or `[report]`) names -- summary, kinds, failures, slowest, metrics, resources, flaky, trend, path, graph, gantt -- over `--scope` (`run`, `program:`, `kind:`), of one run, `--runs a,b`, or every run of `--group G` together | +| `report [--format text\|json\|junit\|html\|md]` | a stored run; `md`: a Markdown report of the sections `--sections` (or `[report]`) names -- summary, kinds, failures, slowest, steps, builds (a host's `builds.ndjson`), margin (cases over four fifths of their limit), budget, ledgers, metrics, resources, flaky, trend, path, graph, gantt -- over `--scope` (`run`, `program:`, `kind:`), of one run, `--runs a,b`, or every run of `--group G` together | | `diff [--before ID] [--after ID]` | newly failing, fixed, slower, faster | | `history --case REF` | one case across the stored runs | | `query` | stored case results by `--status`, `--kind`, `--program`, `--case`, `--tag`, `--days`, `--runs` | | `analyze METRIC` | `hot`, `durations`, `path` (the critical path: the longest weighted path of the run's graph, spec P12.3, with what each step waited for), `failures` (grouped by message), `flaky`, `bench`, `trend` (each case's metrics over the last `--runs N` runs beside their bounds, `--case GLOB`), `cost` | | `graph [--format json\|dot\|mermaid\|gantt]` | sources → programs → cases → resources and artifacts; with `--run ID`, the run's graph of work -- builds, processes, run fixtures and what each waited for (worker, resource, build, fixture) -- the critical path marked, or its schedule as a gantt | -| `gate [--set S]` | a set's gate on a run: no failure, the budget kept, bench verdicts that pass | +| `gate [--set S]` | a set's gate on a run: no failure, the set's budget kept, no case over `[budget] case_cpu` without a waiver, every ledger's requirement met or waived, no waiver that excuses nothing, bench verdicts that pass | +| `coverage spread` | how widely the cases of the coverage map share files and functions: the median share, which decides whether selecting by a finer granularity selects fewer cases | A run is named by its identifier, `last`, or `last-N`. diff --git a/docs/config.md b/docs/config.md index 0e3e7d1..f9fedb7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -138,6 +138,27 @@ shard = "15m" # the expected time of one shard max_shards = 8 unknown = "30s" # the expected time of a case a host never ran +# The processor time a case may take (0.11.0), judged by `mtest gate` from the case's `cpu_ms` (its +# cpu.user and cpu.sys metrics): a case over it fails the gate unless a waiver names it, with a reason; +# a waiver that excuses nothing fails the gate too, so the list only shrinks. +[budget] +case_cpu = "2m" +[[budget.waiver]] +case = "e2e/openkal::trees/955*" # a glob over :: +reason = "builds a platform's runtime for three configurations" + +# A ledger (0.11.0): requirements, one per line of a file or of a command's output (`#` begins a +# comment; `${NAME}` in the command is the variable's value). Each must be covered, over the run and +# the runs of its group, by a passing case's clause (`evidence = "clauses"`, P6.4) or by an item of +# the product's surface a passing case used (`evidence = "exercised"`, P5.6), or be excused by a +# waiver. `mtest gate` judges it and `mtest report --sections ledgers` writes its table. +[ledger.surface] +command = ["${MCPP}", "emit", "surface"] +evidence = "exercised" +[[ledger.surface.waiver]] +item = "flag build --no-cache" +reason = "no case can observe it apart from the cache's" + [sets.fast] # sets.md tags = ["fast"] include = ["smoke"] diff --git a/engine/mcpp.toml b/engine/mcpp.toml index 3c80487..dc947f7 100644 --- a/engine/mcpp.toml +++ b/engine/mcpp.toml @@ -10,7 +10,7 @@ [package] namespace = "mtest" name = "mtest" -version = "0.10.0" +version = "0.11.0" description = "The mtest engine: run, schedule, store, explain and analyse tests of any language" license = "Apache-2.0" authors = ["mcpp-community"] @@ -30,7 +30,7 @@ main = "src/main.cpp" [dependencies] mcpplibs.mtest.manifest = { path = "../manifest" } # the static case manifest -mcpplibs.os = "^0.4.1" +mcpplibs.os = "^0.4.7" mcpplibs.cmdline = "0.0.2" nlohmann.json = "3.12.0" marzer.tomlplusplus = "3.4.0" diff --git a/engine/src/analysis.cpp b/engine/src/analysis.cpp index f6eec78..54ca24f 100644 --- a/engine/src/analysis.cpp +++ b/engine/src/analysis.cpp @@ -1,9 +1,13 @@ module mcpplibs.mtest.engine.analysis; import std; +import mcpplibs.os; +import mcpplibs.mtest.engine.select; namespace mcpplibs::mtest::engine { +namespace os = mcpplibs::os; + namespace { double median(std::vector v) { @@ -324,8 +328,174 @@ judge_bench(const run_record& r, const std::optional& baseline, cons return out; } +namespace { + +// The run and, when it is a part of a group (the jobs of one CI run), the other parts. +std::vector group_of(const store& s, const run_record& r) { + std::vector out{r}; + if (r.group.empty()) + return out; + for (const auto& summary : runs(s)) + if (summary.group == r.group && summary.id != r.id) + if (auto other = load_run(s, summary.id)) + out.push_back(std::move(*other)); + return out; +} + +bool waived(const std::vector& ws, std::string_view what) { + return std::ranges::any_of(ws, [&](const waiver& w) { return glob_match(w.what, what); }); +} + +// A ledger's requirements: the lines of its file or of its command's output, each trimmed; empty +// lines and lines starting with `#` are none. +std::expected, std::string> requirements(const config& c, + const ledger_spec& l) { + std::string text; + if (!l.file.empty()) { + auto f = os::fs::read(l.file.is_absolute() ? l.file : c.root / l.file); + if (!f) + return std::unexpected( + std::format("ledger {}: {} cannot be read", l.name, os::fs::utf8(l.file))); + text = std::move(*f); + } else { + std::vector argv; + for (auto word : l.command) { + // `${NAME}`: the variable's value. + for (std::size_t at = word.find("${"); at != std::string::npos; + at = word.find("${", at)) { + const auto end = word.find('}', at); + if (end == std::string::npos) + break; + const auto value = os::env::get(word.substr(at + 2, end - at - 2)).value_or(""); + word.replace(at, end - at + 1, value); + at += value.size(); + } + argv.push_back(std::move(word)); + } + os::process::spawn_options o; + o.program = os::fs::from_utf8(argv.front()); + o.args.assign(argv.begin() + 1, argv.end()); + o.cwd = c.root; + auto done = os::process::run(std::move(o), os::process::bounds{}); + if (!done || !done->status.success()) + return std::unexpected(std::format("ledger {}: `{}` did not give its requirements{}", + l.name, argv.front(), + done ? ": " + done->err.substr(0, 200) : "")); + text = std::move(done->out); + } + std::vector out; + for (auto part : std::views::split(std::string_view(text), '\n')) { + std::string line(part.begin(), part.end()); + while (!line.empty() && (line.back() == '\r' || line.back() == ' ')) + line.pop_back(); + if (!line.empty() && !line.starts_with('#')) + out.push_back(std::move(line)); + } + return out; +} + +} // namespace + +json ledger_table(const config& c, const store& s, const run_record& r) { + json out = json::array(); + const auto all = group_of(s, r); + for (const auto& l : c.ledgers) { + std::set covered; + for (const auto& run : all) + for (const auto& cr : run.cases) + if (cr.status == "pass" || cr.status == "xfail") + for (const auto& item : l.evidence == "exercised" ? cr.exercised : cr.clauses) + covered.insert(item); + json one = {{"ledger", l.name}, {"evidence", l.evidence}}; + auto reqs = requirements(c, l); + if (!reqs) { + one["error"] = reqs.error(); + out.push_back(std::move(one)); + continue; + } + json missing = json::array(), excused = json::array(), stale = json::array(); + std::size_t met = 0; + for (const auto& req : *reqs) { + const bool is_covered = covered.contains(req); + const bool is_waived = waived(l.waivers, req); + if (is_covered && is_waived) + stale.push_back(req); + else if (is_covered) + ++met; + else if (is_waived) + excused.push_back(req); + else + missing.push_back(req); + } + for (const auto& w : l.waivers) + if (std::ranges::none_of(*reqs, [&](const std::string& req) { + return glob_match(w.what, req); + })) + stale.push_back(w.what); + one["requirements"] = reqs->size(); + one["met"] = met; + one["missing"] = missing; + one["waived"] = excused; + one["stale_waivers"] = stale; + out.push_back(std::move(one)); + } + return out; +} + +json budget_overruns(const config& c, const run_record& r) { + json out = json::array(); + if (!c.budget.case_cpu) + return out; + const double limit = double(c.budget.case_cpu->count()); + std::set over; + for (const auto& cr : r.cases) { + if (cr.cpu_ms < 0 || cr.cpu_ms <= limit) + continue; + const auto key = cr.program + "::" + cr.id; + over.insert(key); + out.push_back({{"case", key}, + {"cpu_ms", cr.cpu_ms}, + {"limit_ms", limit}, + {"machine", r.machine}, + {"waived", waived(c.budget.waivers, key)}}); + } + for (const auto& w : c.budget.waivers) + if (std::ranges::none_of(over, + [&](const std::string& k) { return glob_match(w.what, k); }) && + std::ranges::any_of(r.cases, [&](const case_result& cr) { + return cr.cpu_ms >= 0 && glob_match(w.what, cr.program + "::" + cr.id); + })) + out.push_back({{"case", w.what}, {"stale_waiver", true}, {"reason", w.reason}}); + return out; +} + json gate(const config& c, const store& s, const run_record& r, std::string_view set) { json reasons = json::array(); + for (const auto& o : budget_overruns(c, r)) { + if (o.value("stale_waiver", false)) + reasons.push_back( + std::format("budget waiver for {} excuses nothing: the case is within " + "its budget, remove the waiver", + o.value("case", ""))); + else if (!o.value("waived", false)) + reasons.push_back(std::format("{} took {:.1f} s of processor time on {}, over the case " + "budget of {:.0f} s -- make it cheaper or waive it with " + "a reason ([[budget.waiver]])", + o.value("case", ""), o.value("cpu_ms", 0.0) / 1000, + o.value("machine", ""), o.value("limit_ms", 0.0) / 1000)); + } + for (const auto& l : ledger_table(c, s, r)) { + if (l.contains("error")) { + reasons.push_back(l["error"].get()); + continue; + } + for (const auto& m : l["missing"]) + reasons.push_back(std::format("ledger {}: {} has no passing case and no waiver", + l.value("ledger", ""), m.get())); + for (const auto& w : l["stale_waivers"]) + reasons.push_back(std::format("ledger {}: the waiver {} excuses nothing, remove it", + l.value("ledger", ""), w.get())); + } for (const auto& cr : r.cases) if (failing(cr.status) && !cr.quarantined) reasons.push_back(std::format("{}::{} {}{}", cr.program, cr.id, cr.status, diff --git a/engine/src/analysis.cppm b/engine/src/analysis.cppm index 6baf528..c4f93a4 100644 --- a/engine/src/analysis.cppm +++ b/engine/src/analysis.cppm @@ -10,6 +10,10 @@ export namespace mcpplibs::mtest::engine { // The slowest cases of a run. json hot(const run_record& r, std::size_t top); +// Each ledger's requirements, met, missing and waived over the run and its group (0.11.0). +json ledger_table(const config& c, const store& s, const run_record& r); +// The cases over `[budget] case_cpu`, waived or not, and the waivers that excuse nothing (0.11.0). +json budget_overruns(const config& c, const run_record& r); // Where a run's time went: by kind and by program, the share each program had of the wall clock, // and the critical path (the longest program: no schedule finishes before it) against the wall // clock. diff --git a/engine/src/commands.cpp b/engine/src/commands.cpp index 01924a8..43c8bf2 100644 --- a/engine/src/commands.cpp +++ b/engine/src/commands.cpp @@ -340,6 +340,10 @@ answer cmd_run(const std::filesystem::path& cwd, const json& p, o.jobs = p.value("jobs", 0); if (p.contains("seed") && p["seed"].is_number_unsigned()) o.seed = p["seed"].get(); + o.jitter_ms = std::max(0, p.value("jitter", 0)); + if (p.contains("jitter_seed") && p["jitter_seed"].is_number_integer() && + p["jitter_seed"].get() >= 0) + o.jitter_seed = p["jitter_seed"].get(); o.run_under = strings(p, "run_under"); if (auto e = p.find("env"); e != p.end() && e->is_object()) for (auto& item_ : e->items()) @@ -418,8 +422,32 @@ answer cmd_coverage(const std::filesystem::path& cwd, const json& p) { std::format("coverage map of {} taken in: {} case(s)\n", m->commit.substr(0, 12), m->cases.size())}; } + if (action == "spread") { + // How widely the cases share what they executed: the measure that decides whether a + // selection at a finer granularity selects fewer cases (0.11.0). + auto m = read_coverage(file); + if (!m) + return usage(m.error()); + const auto sp = spread_of(*m); + return {0, + {{"cases", sp.cases}, + {"files", sp.files}, + {"functions", sp.functions}, + {"median_file_share", sp.median_file_share}, + {"median_function_share", sp.median_function_share}, + {"conclusion", + std:: + format("of {} case(s), the median file is executed by {:.0f}% and the median " + "function by {:.0f}%", + sp.cases, sp.median_file_share * 100, + sp.median_function_share * 100)}}, + std::format("{} case(s); {} file(s), median share {:.1f}%; {} function(s), median " + "share {:.1f}%\n", + sp.cases, sp.files, sp.median_file_share * 100, sp.functions, + sp.median_function_share * 100)}; + } if (!action.empty() && action != "show") - return usage(std::format("coverage {}: the actions are show and import", action)); + return usage(std::format("coverage {}: the actions are show, import and spread", action)); auto m = read_coverage(file); if (!m) return {0, @@ -431,18 +459,24 @@ answer cmd_coverage(const std::filesystem::path& cwd, const json& p) { std::string text = std::format("coverage map of {} at {}{}\n", m->commit.substr(0, 12), m->time, fresh ? "" : " (older than [coverage] max_age: not read)"); const auto only = strings(p, "case"); + json functions = json::object(); for (const auto& [k, files] : m->cases) { if (!only.empty() && !std::ranges::any_of(only, [&](const std::string& g) { return glob_match(g, k); })) continue; cases[k] = files; - text += std::format("{}: {} file(s)\n", k, files.size()); + const auto fns = m->functions.find(k); + if (fns != m->functions.end()) + functions[k] = fns->second; + text += std::format("{}: {} file(s), {} function(s)\n", k, files.size(), + fns == m->functions.end() ? 0 : fns->second.size()); } return {0, {{"commit", m->commit}, {"time", m->time}, {"fresh", fresh}, {"cases", cases}, + {"functions", functions}, {"conclusion", std::format("{} case(s) mapped at {}{}", m->cases.size(), m->commit.substr(0, 12), fresh ? "" : "; older than max_age")}}, @@ -692,6 +726,7 @@ report_options report_options_of(const context& ctx, const json& p) { o.top = p.value("top", ctx.cfg.report.top); o.max_nodes = p.value("max_nodes", ctx.cfg.report.max_nodes); o.title = text_of(p, "title"); + o.cfg = &ctx.cfg; return o; } @@ -1798,6 +1833,11 @@ json catalog() { "record, its failures and a conclusion.", with({{"jobs", {{"type", "integer"}, {"description", "programs at once"}}}, {"seed", {{"type", "integer"}, {"description", "the seed of anything random"}}}, + {"jitter", + {{"type", "integer"}, + {"description", "the most milliseconds a program waits at a contention point " + "(SPEC-017 O12): a search for an order defect, never a verdict"}}}, + {"jitter_seed", {{"type", "integer"}, {"description", "the seed of the waits"}}}, {"fail_fast", {{"type", "boolean"}, {"description", "stop at the first failure"}}}, {"resume", {{"type", "string"}, @@ -1915,8 +1955,9 @@ json catalog() { {"until", {{"type", "string"}}}}), true); add("coverage", - "The coverage map affected selection reads: show it, or import one (a CI artifact).", - plain({{"action", {{"enum", {"show", "import"}}}}, + "The coverage map affected selection reads: show it, import one (a CI artifact), or " + "measure how widely its cases share files and functions (spread).", + plain({{"action", {{"enum", {"show", "import", "spread"}}}}, {"from", {{"type", "string"}, {"description", "import: a coverage map file"}}}, {"case", {{"type", "array"}, {"items", {{"type", "string"}}}}}})); add("store", "The store: gc, verify, or import another store.", @@ -1968,7 +2009,7 @@ json catalog() { plain({{"out", {{"type", "string"}}}, {"argv", {{"type", "array"}, {"items", {{"type", "string"}}}}}}), true); - return {{"schema", "mtest.catalog/1"}, {"version", "0.10.0"}, {"tools", tools}}; + return {{"schema", "mtest.catalog/1"}, {"version", "0.11.0"}, {"tools", tools}}; } } // namespace mcpplibs::mtest::engine diff --git a/engine/src/config.cpp b/engine/src/config.cpp index 8f74680..3133dd8 100644 --- a/engine/src/config.cpp +++ b/engine/src/config.cpp @@ -240,9 +240,9 @@ std::expected load_config(const std::filesystem::path& dir) return std::unexpected(std::format("{}: {}", os::fs::utf8(file), e.description())); } known(t, "mtest.toml", - {"project", "programs", "discover", "host", "sets", "policy", "retry", "regression", - "store", "trace", "cases", "coverage", "static", "run", "monitor", "report", "fixtures", - "capabilities", "ci"}, + {"project", "programs", "discover", "host", "sets", "policy", "retry", + "regression", "store", "trace", "cases", "coverage", "static", "run", + "monitor", "report", "fixtures", "capabilities", "ci", "budget", "ledger"}, c); if (auto p = t["project"].as_table()) { known(*p, "project", {"store", "jobs"}, c); @@ -466,6 +466,62 @@ std::expected load_config(const std::filesystem::path& dir) c.ci_plan.unknown = *d; } } + // Waivers: `[[budget.waiver]]` and `[[ledger..waiver]]`, each a `case` or an `item` and + // its `reason`. + auto waivers_of = [&](toml::node_view n, std::string_view where, + std::string_view key) -> std::expected, std::string> { + std::vector out; + auto a = n.as_array(); + if (!a) + return out; + for (auto& e : *a) { + auto w = e.as_table(); + if (!w) + return std::unexpected(std::format("{}.waiver must be an array of tables", where)); + known(*w, std::format("{}.waiver", where), {std::string(key), "reason"}, c); + waiver one{(*w)[key].value_or(std::string{}), (*w)["reason"].value_or(std::string{})}; + if (one.what.empty() || one.reason.empty()) + return std::unexpected( + std::format("{}.waiver needs `{}` and `reason`: a waiver says why", where, + key)); + out.push_back(std::move(one)); + } + return out; + }; + if (auto b = t["budget"].as_table()) { + known(*b, "budget", {"case_cpu", "waiver"}, c); + c.budget.case_cpu = duration_of((*b)["case_cpu"]); + auto w = waivers_of((*b)["waiver"], "budget", "case"); + if (!w) + return std::unexpected(w.error()); + c.budget.waivers = std::move(*w); + } + if (auto ls = t["ledger"].as_table()) + for (const auto& [name, node] : *ls) { + auto lt = node.as_table(); + if (!lt) + return std::unexpected(std::format("ledger.{} must be a table", name.str())); + const auto where = std::format("ledger.{}", name.str()); + known(*lt, where, {"file", "command", "evidence", "waiver"}, c); + ledger_spec l; + l.name = std::string(name.str()); + if (auto f = (*lt)["file"].value()) + l.file = os::fs::from_utf8(*f); + l.command = strings((*lt)["command"]); + l.evidence = (*lt)["evidence"].value_or(std::string("clauses")); + if (l.evidence != "clauses" && l.evidence != "exercised") + return std::unexpected( + std::format("{}.evidence is `clauses` or `exercised`, not `{}`", where, + l.evidence)); + if (l.file.empty() == l.command.empty()) + return std::unexpected( + std::format("{} states one source: `file` or `command`", where)); + auto w = waivers_of((*lt)["waiver"], where, "item"); + if (!w) + return std::unexpected(w.error()); + l.waivers = std::move(*w); + c.ledgers.push_back(std::move(l)); + } if (auto fs = t["fixtures"].as_table()) for (const auto& [name, node] : *fs) { auto ft = node.as_table(); diff --git a/engine/src/config.cppm b/engine/src/config.cppm index edf457b..b52552c 100644 --- a/engine/src/config.cppm +++ b/engine/src/config.cppm @@ -92,8 +92,9 @@ struct monitor_spec { // `[report]`: what `mtest report --format md` writes when the command does not say. struct report_spec { - std::vector sections{"summary", "failures", "slowest", "metrics", - "resources", "path", "gantt"}; + std::vector sections{"summary", "failures", "slowest", "steps", + "builds", "margin", "budget", "ledgers", + "metrics", "resources", "path", "gantt"}; std::string scope = "run"; // run, program:, kind:, group int top = 10; // rows of the ranked tables int max_nodes = 60; // nodes of a drawn graph @@ -144,6 +145,34 @@ struct ci_plan_spec { std::chrono::milliseconds unknown{std::chrono::seconds{30}}; // a case with no history }; +// A case excused from a rule of the gate, with its reason (0.11.0): a waiver names a case (a glob +// over `::`) or an item of a ledger, and only shrinks -- one that excuses nothing +// fails the gate. +struct waiver { + std::string what; + std::string reason; +}; + +// `[budget]` (0.11.0): the processor time a case may take, judged by `mtest gate` from the +// case's `cpu_ms` (P12.4); a case over it fails the gate unless a waiver names it. +struct budget_spec { + std::optional case_cpu; + std::vector waivers; +}; + +// `[ledger.]` (0.11.0): requirements, one per line of a file or of a command's output, each +// of which a passing case must cover -- by a clause it states (`evidence = "clauses"`, P6.4) or +// by an item of the product's surface it used (`evidence = "exercised"`, P5.6) -- or a waiver +// must excuse. `mtest gate` judges it over the run and the runs of its group; `mtest report` +// writes its table. +struct ledger_spec { + std::string name; + std::filesystem::path file; + std::vector command; // `${NAME}` is the variable's value + std::string evidence = "clauses"; + std::vector waivers; +}; + struct config { std::filesystem::path root; // the directory of mtest.toml, or the working directory std::filesystem::path store = ".mtest"; @@ -172,6 +201,8 @@ struct config { capabilities_spec capabilities; ci_coverage_spec ci_coverage; ci_plan_spec ci_plan; + budget_spec budget; + std::vector ledgers; std::vector warnings; // keys this engine does not know }; diff --git a/engine/src/coverage.cpp b/engine/src/coverage.cpp index 583ecbd..ba475be 100644 --- a/engine/src/coverage.cpp +++ b/engine/src/coverage.cpp @@ -9,8 +9,9 @@ namespace os = mcpplibs::os; namespace { -constexpr std::uint32_t kGcdaMagic = 0x67636461; // "gcda" -constexpr std::uint32_t kArcsTag = 0x01a10000; // GCOV_TAG_COUNTER_BASE: the arc counters +constexpr std::uint32_t kGcdaMagic = 0x67636461; // "gcda" +constexpr std::uint32_t kArcsTag = 0x01a10000; // GCOV_TAG_COUNTER_BASE: the arc counters +constexpr std::uint32_t kFunctionTag = 0x01000000; // GCOV_TAG_FUNCTION: ident, checksums std::uint32_t swapped(std::uint32_t v) { return (v >> 24) | ((v >> 8) & 0xff00) | ((v << 8) & 0xff0000) | (v << 24); @@ -78,8 +79,97 @@ std::vector executed_sources(const std::filesystem::path& dir, return {files.begin(), files.end()}; } +// The functions one program's run executed, `#`. +std::vector executed_functions(const std::filesystem::path& dir, + const std::map& sources) { + std::set out; + std::error_code ec; + for (std::filesystem::recursive_directory_iterator it(dir, ec), end; it != end; + it.increment(ec)) { + if (it->path().extension() != ".gcda") + continue; + const auto rel = std::filesystem::relative(it->path(), dir, ec); + auto s = sources.find(stem_of(rel)); + if (s == sources.end()) + continue; + for (auto ident : gcda_functions(it->path())) + out.insert(std::format("{}#{}", s->second, ident)); + } + return {out.begin(), out.end()}; +} + +double median_of(std::vector v) { + if (v.empty()) + return 0; + std::ranges::sort(v); + return v.size() % 2 ? v[v.size() / 2] : (v[v.size() / 2 - 1] + v[v.size() / 2]) / 2; +} + +std::pair +spread_over(const std::map>& by_case) { + std::map count; + for (const auto& [_, items] : by_case) + for (const auto& item : items) + ++count[item]; + std::vector shares; + for (const auto& [_, n] : count) + shares.push_back(by_case.empty() ? 0 : double(n) / double(by_case.size())); + return {count.size(), median_of(std::move(shares))}; +} + } // namespace +std::vector gcda_functions(const std::filesystem::path& gcda) { + std::vector out; + std::ifstream in(gcda, std::ios::binary); + std::vector bytes((std::istreambuf_iterator(in)), {}); + if (bytes.size() < 16 || bytes.size() % 4) + return out; + std::vector words(bytes.size() / 4); + std::memcpy(words.data(), bytes.data(), bytes.size()); + const bool swap = words[0] != kGcdaMagic; + if (swap && swapped(words[0]) != kGcdaMagic) + return out; + auto word = [&](std::size_t i) { return swap ? swapped(words[i]) : words[i]; }; + std::optional current; + bool ran = false; + auto close = [&] { + if (current && ran) + out.push_back(*current); + current.reset(); + ran = false; + }; + for (std::size_t i = 4; i + 2 <= words.size();) { + const auto tag = word(i); + const auto length = static_cast(word(i + 1)); + i += 2; + if (tag == 0) + break; + if (tag == kFunctionTag) { + close(); + if (length > 0 && i < words.size()) + current = word(i); + } + if (length <= 0) + continue; + const auto n = static_cast(length) / 4; + if (tag == kArcsTag) + for (std::size_t k = i; k < i + n && k < words.size(); ++k) + ran = ran || word(k) != 0; + i += n; + } + close(); + return out; +} + +coverage_spread spread_of(const coverage_map& m) { + coverage_spread out; + out.cases = m.cases.size(); + std::tie(out.files, out.median_file_share) = spread_over(m.cases); + std::tie(out.functions, out.median_function_share) = spread_over(m.functions); + return out; +} + std::filesystem::path coverage_file(const store& s) { return s.dir / "coverage.json"; } @@ -105,6 +195,15 @@ std::expected read_coverage(const std::filesystem::pa files.push_back(f.get()); std::ranges::sort(files); } + if (j.contains("functions") && j["functions"].is_object()) + for (auto it = j["functions"].begin(); it != j["functions"].end(); ++it) { + auto& fns = m.functions[it.key()]; + if (it.value().is_array()) + for (const auto& f : it.value()) + if (f.is_string()) + fns.push_back(f.get()); + std::ranges::sort(fns); + } return m; } @@ -112,11 +211,11 @@ void write_coverage(const std::filesystem::path& file, const coverage_map& m) { json cases = json::object(); for (const auto& [k, files] : m.cases) cases[k] = files; - json j = {{"format", kCoverageFormat}, - {"commit", m.commit}, - {"time", m.time}, - {"epoch", m.epoch}, - {"cases", cases}}; + json functions = json::object(); + for (const auto& [k, fns] : m.functions) + functions[k] = fns; + json j = {{"format", kCoverageFormat}, {"commit", m.commit}, {"time", m.time}, + {"epoch", m.epoch}, {"cases", cases}, {"functions", functions}}; std::error_code ec; std::filesystem::create_directories(file.parent_path(), ec); const auto tmp = file.string() + ".tmp"; @@ -193,8 +292,11 @@ collect_coverage(const config& c, .count(); for (const auto& [dir, cases] : dirs) { const auto files = executed_sources(dir, *sources); - for (const auto& k : cases) + const auto fns = executed_functions(dir, *sources); + for (const auto& k : cases) { m.cases[k] = files; + m.functions[k] = fns; + } } return m; } diff --git a/engine/src/coverage.cppm b/engine/src/coverage.cppm index ff3dfe9..4f0abf4 100644 --- a/engine/src/coverage.cppm +++ b/engine/src/coverage.cppm @@ -19,8 +19,23 @@ struct coverage_map { std::string commit, time; // the run it was measured from std::int64_t epoch = 0; // when, in seconds since 1970: what `max_age` is measured against std::map> cases; // program::case -> files from the root + // program::case -> the functions it executed, `#` with GCC's function ident + // (0.11.0): what a selection by function would read, and how widely each function is shared. + std::map> functions; }; +// How widely the cases of a map share what they executed (0.11.0): for each file and each function, +// the share of the cases that executed it, and the medians. A selection at a granularity is worth +// it only where the median share is small. +struct coverage_spread { + std::size_t cases = 0, files = 0, functions = 0; + double median_file_share = 0, median_function_share = 0; +}; +coverage_spread spread_of(const coverage_map& m); + +// The functions whose arc counters ran in one `.gcda`, by GCC's ident (0.11.0). +std::vector gcda_functions(const std::filesystem::path& gcda); + std::filesystem::path coverage_file(const store& s); std::expected read_coverage(const std::filesystem::path& file); void write_coverage(const std::filesystem::path& file, const coverage_map& m); diff --git a/engine/src/exec.cpp b/engine/src/exec.cpp index 09a0e60..e45d371 100644 --- a/engine/src/exec.cpp +++ b/engine/src/exec.cpp @@ -282,6 +282,13 @@ report_reading read_report(std::string_view text, std::string_view program) { if (x.is_number()) m.samples.push_back(x.get()); at(id).metrics.push_back(std::move(m)); + } else if (kind == "step") { + at(id).steps.push_back(step_from(e)); + } else if (kind == "exercise") { + auto& c = at(id); + for (auto& item : strings_of(e, "items")) + if (std::ranges::find(c.exercised, item) == c.exercised.end()) + c.exercised.push_back(std::move(item)); } else if (kind == "artifact") { at(id).artifacts.push_back(e.value("path", "")); } else if (kind == "fixture") { @@ -291,6 +298,16 @@ report_reading read_report(std::string_view text, std::string_view program) { c.status = e.value("status", "fail"); c.ms = e.value("ms", 0.0); c.reason = e.value("reason", ""); + std::ranges::sort(c.exercised); + double cpu = 0; + bool told = false; + for (const auto& m : c.metrics) + if (m.name == "cpu.user" || m.name == "cpu.sys") { + cpu += m.value; + told = true; + } + if (told) + c.cpu_ms = cpu; } else if (kind == "missing") { auto& c = at(id); c.status = "fail"; diff --git a/engine/src/main.cpp b/engine/src/main.cpp index e0573b7..064ad41 100644 --- a/engine/src/main.cpp +++ b/engine/src/main.cpp @@ -98,7 +98,7 @@ const json* tool_named(const json& tools, std::string_view name) { cmdline::App application(const json& tools) { auto app = cmdline::App("mtest") - .version("0.10.0") + .version("0.11.0") .description("run tests of any language, and ask about their runs") .option(cmdline::Option("json").global().help("print the answer's data")); for (const auto& t : tools) @@ -253,7 +253,7 @@ int main(int argc, char** argv) { const std::string name(parsed->subcommand_name()); const auto& sub = parsed->subcommand()->get(); if (name == "version") { - std::println("mtest 0.10.0"); + std::println("mtest 0.11.0"); return 0; } const auto cwd = std::filesystem::current_path(); diff --git a/engine/src/model.cpp b/engine/src/model.cpp index c863564..b1beada 100644 --- a/engine/src/model.cpp +++ b/engine/src/model.cpp @@ -1,9 +1,12 @@ module mcpplibs.mtest.engine.model; import std; +import mcpplibs.os; namespace mcpplibs::mtest::engine { +namespace os = mcpplibs::os; + namespace { template std::vector strings(const json& j, const char* key) { @@ -125,9 +128,45 @@ json to_json(const case_result& c) { j["attempts"] = c.attempts; if (c.started_ms >= 0) j["started_ms"] = c.started_ms; + if (c.cpu_ms >= 0) + j["cpu_ms"] = c.cpu_ms; + if (!c.steps.empty()) { + j["steps"] = json::array(); + for (const auto& st : c.steps) { + json one = {{"name", st.name}, {"t", st.t}, {"ms", st.ms}}; + if (st.cpu_ms) + one["cpu_ms"] = *st.cpu_ms; + if (st.status) + one["status"] = *st.status; + if (!st.build.empty()) + one["build"] = st.build; + j["steps"].push_back(std::move(one)); + } + } + if (!c.exercised.empty()) + j["exercised"] = c.exercised; + if (c.limit_ms >= 0) + j["limit_ms"] = c.limit_ms; return j; } +build_edge build_edge_from(const json& j) { + build_edge b; + b.build = str(j, "build"); + b.output = str(j, "output"); + b.package = str(j, "package"); + b.program = str(j, "program"); + b.started_ms = j.value("started_ms", 0.0); + b.ms = j.value("ms", 0.0); + if (auto it = j.find("cpu_ms"); it != j.end() && it->is_number()) + b.cpu_ms = it->get(); + if (auto it = j.find("peak_bytes"); it != j.end() && it->is_number()) + b.peak_bytes = it->get(); + b.cached = j.value("cached", false); + b.units = j.value("units", 0); + return b; +} + namespace { // A process as the timeout recorded it; an empty field is one the system did not tell. @@ -256,17 +295,38 @@ fixture_run fixture_run_from(const json& j) { return {str(j, "name"), str(j, "program"), j.value("started_ms", 0.0), j.value("ready_ms", 0.0), j.value("ended_ms", 0.0), str(j, "status")}; } +std::string machine_class() { + if (auto model = os::machine::processor_model(); model && !model->empty()) + return *model; + return std::format("{}/{}", os::machine::arch(), os::machine::processors().logical); +} + json to_json(const run_record& r) { - return {{"schema", "mtest.run/1"}, {"id", r.id}, - {"started", r.started}, {"finished", r.finished}, - {"status", r.status}, {"system", r.system}, - {"arch", r.arch}, {"machine", r.machine}, - {"commit", r.commit}, {"set", r.set}, - {"selection", r.selection}, {"ms", r.ms}, - {"jobs", r.jobs}, {"counts", r.counts}, - {"group", r.group}, {"monitor", r.monitor}, - {"host", r.host}, {"capabilities", r.capabilities}, - {"resumed", r.resumed}, {"programs", r.programs.size()}, + return {{"schema", "mtest.run/1"}, + {"id", r.id}, + {"processors", r.processors}, + {"memory", r.memory}, + {"budget", r.budget}, + {"jitter", + r.jitter_ms > 0 ? json{{"max_ms", r.jitter_ms}, {"seed", r.jitter_seed}} : json()}, + {"started", r.started}, + {"finished", r.finished}, + {"status", r.status}, + {"system", r.system}, + {"arch", r.arch}, + {"machine", r.machine}, + {"commit", r.commit}, + {"set", r.set}, + {"selection", r.selection}, + {"ms", r.ms}, + {"jobs", r.jobs}, + {"counts", r.counts}, + {"group", r.group}, + {"monitor", r.monitor}, + {"host", r.host}, + {"capabilities", r.capabilities}, + {"resumed", r.resumed}, + {"programs", r.programs.size()}, {"cases", r.cases.size()}}; } @@ -305,6 +365,18 @@ case_info case_info_from(const json& j) { c.reason = str(j, "reason"); return c; } +step_info step_from(const json& j) { + step_info st; + st.name = str(j, "name"); + st.t = j.value("t", 0.0); + st.ms = j.value("ms", 0.0); + if (auto it = j.find("cpu_ms"); it != j.end() && it->is_number()) + st.cpu_ms = it->get(); + if (auto it = j.find("status"); it != j.end() && it->is_number_integer()) + st.status = it->get(); + st.build = str(j, "build"); + return st; +} case_result case_result_from(const json& j) { case_result c; c.program = str(j, "program"); @@ -343,6 +415,12 @@ case_result case_result_from(const json& j) { c.fixtures.push_back({str(f, "name"), f.value("ms", 0.0)}); c.diagnosis = str(j, "diagnosis"); c.bounds = bounds_from(j); + c.cpu_ms = j.value("cpu_ms", -1.0); + if (auto it = j.find("steps"); it != j.end() && it->is_array()) + for (const auto& st : *it) + c.steps.push_back(step_from(st)); + c.exercised = strings(j, "exercised"); + c.limit_ms = j.value("limit_ms", -1.0); c.started_ms = j.value("started_ms", -1.0); return c; } @@ -393,6 +471,13 @@ run_record run_record_from(const json& j) { r.system = str(j, "system"); r.arch = str(j, "arch"); r.machine = str(j, "machine"); + r.processors = j.value("processors", 0); + r.memory = j.value("memory", std::uint64_t{0}); + r.budget = j.value("budget", 0); + if (auto jt = j.find("jitter"); jt != j.end() && jt->is_object()) { + r.jitter_ms = jt->value("max_ms", 0); + r.jitter_seed = jt->value("seed", std::uint64_t{0}); + } r.commit = str(j, "commit"); r.set = str(j, "set"); r.selection = str(j, "selection"); diff --git a/engine/src/model.cppm b/engine/src/model.cppm index 1318e16..9a6554f 100644 --- a/engine/src/model.cppm +++ b/engine/src/model.cppm @@ -103,6 +103,15 @@ struct metric_info { std::vector samples; }; +// A step of a case (P5.6, 0.11.0): a command an end-to-end case ran. +struct step_info { + std::string name; + double t = 0, ms = 0; + std::optional cpu_ms; + std::optional status; + std::string build; // the record of the build the step was, when it was one +}; + // One case's result. An L0 program is one case whose identifier is the program's name. struct case_result { std::string program, id, kind = "unit"; @@ -121,6 +130,23 @@ struct case_result { // When it began, in milliseconds after the run began (its program's start and its `begin`'s // `t`); negative when not known. double started_ms = -1; + // Its processor time: the `cpu.user` and `cpu.sys` metrics it reported (P6.6), else the time + // its program used while it ran alone in it; negative when not known (0.11.0). + double cpu_ms = -1; + std::vector steps; // P5.6 (0.11.0) + std::vector exercised; // the items of the product's surface it used, sorted + // The time it may take, as the runner applied it (SPEC-017 O5); negative when not known. + double limit_ms = -1; +}; + +// A build the runner made for a run (B4 `builds.ndjson`, 0.11.0): an edge it ran, or a package the +// cache served (`cached`, with its `units`). +struct build_edge { + std::string build, output, package, program; + double started_ms = 0, ms = 0; + std::optional cpu_ms, peak_bytes; + bool cached = false; + int units = 0; }; // A thread of a process, as the engine saw it at a timeout (0.10.0). @@ -192,12 +218,20 @@ struct run_record { std::string id; std::string started, finished; // ISO 8601, UTC std::string status; // pass fail infra cancelled + // `machine`: the machine's class (P12.4, 0.11.0), the processor's model, else + // `/`; durations are compared within a class. std::string system, arch, machine, commit, set, selection; + int processors = 0; + std::uint64_t memory = 0; + int budget = 0; // O11: the slots of the run's budget; 0: none (0.11.0) + int jitter_ms = 0; // O12: the most a program waited at a contention point + std::uint64_t jitter_seed = 0; double ms = 0; int jobs = 1; std::vector programs; std::vector cases; std::vector fixtures; + std::vector builds; // read from a host's builds.ndjson (0.11.0) std::map counts; // Runs of one group are parts of one whole (the jobs of a CI run, MTEST_GROUP), reported // together. @@ -211,6 +245,10 @@ struct run_record { std::string resumed; }; +// This machine's class (P12.4): the processor's model as the system names it, else the +// architecture and the number of processors. +std::string machine_class(); + // JSON as text, a byte that is not UTF-8 (a program's output in a Windows code page) written as // U+FFFD rather than refused. std::string dumped(const json& j, int indent = -1); @@ -227,6 +265,8 @@ json to_json(const fixture_run& f); fixture_run fixture_run_from(const json& j); case_info case_info_from(const json& j); +step_info step_from(const json& j); +build_edge build_edge_from(const json& j); case_result case_result_from(const json& j); timeout_info timeout_info_from(const json& j); program_result program_result_from(const json& j); diff --git a/engine/src/report.cpp b/engine/src/report.cpp index de8e957..9757ade 100644 --- a/engine/src/report.cpp +++ b/engine/src/report.cpp @@ -139,9 +139,9 @@ std::vector cases_of(const std::vector& runs, } void summary(std::string& md, const std::vector& runs, const report_options& o) { - md += "## Summary\n\n| run | status | system | commit | cases | pass | fail | skip | wall | " - "jobs |\n" - "|---|---|---|---|---:|---:|---:|---:|---:|---:|\n"; + md += "## Summary\n\n| run | status | system | machine | commit | cases | pass | fail | skip | " + "wall | jobs |\n" + "|---|---|---|---|---|---:|---:|---:|---:|---:|---:|\n"; std::map total; double wall = 0; for (const auto& r : runs) { @@ -155,16 +155,17 @@ void summary(std::string& md, const std::vector& runs, const report_ total[k] += v; } wall = std::max(wall, r.ms); - md += std::format("| `{}` | {} | {}-{} | `{}` | {} | {} | {} | {} | {} | {} |\n", r.id, + md += std::format("| `{}` | {} | {}-{} | {} | `{}` | {} | {} | {} | {} | {} | {} |\n", r.id, r.status == "pass" ? "pass" : "**" + r.status + "**", r.system, r.arch, - r.commit.substr(0, 10), cases, n["pass"] + n["xfail"] + n["flaky"], + r.machine.empty() ? "-" : r.machine, r.commit.substr(0, 10), cases, + n["pass"] + n["xfail"] + n["flaky"], n["fail"] + n["xpass"] + n["timeout"], n["skip"], sec(r.ms), r.jobs); } if (runs.size() > 1) { int cases = 0; for (const auto& [k, v] : total) cases += v; - md += std::format("| **all** | | | | {} | {} | {} | {} | {} | |\n", cases, + md += std::format("| **all** | | | | | {} | {} | {} | {} | {} | |\n", cases, total["pass"] + total["xfail"] + total["flaky"], total["fail"] + total["xpass"] + total["timeout"], total["skip"], sec(wall)); @@ -172,6 +173,125 @@ void summary(std::string& md, const std::vector& runs, const report_ md += "\n"; } +// The slowest steps of the cases (P5.6): where an end-to-end case spends its time. +void steps_section(std::string& md, const std::vector& cases, int top) { + struct row { + const case_result* c; + const step_info* st; + }; + std::vector rows; + for (auto* c : cases) + for (const auto& st : c->steps) + rows.push_back({c, &st}); + if (rows.empty()) + return; + std::ranges::sort(rows, std::greater{}, [](const row& r) { return r.st->ms; }); + md += "## Slowest steps\n\n| case | step | time | processor | status | build |\n" + "|---|---|---:|---:|---:|---|\n"; + for (std::size_t i = 0; i < rows.size() && int(i) < top; ++i) { + const auto& st = *rows[i].st; + md += std::format("| `{}` | `{}` | {} | {} | {} | {} |\n", + case_label(rows[i].c->program, rows[i].c->id), cell(st.name, 80), + sec(st.ms), st.cpu_ms ? sec(*st.cpu_ms) : "-", + st.status ? std::to_string(*st.status) : "-", + st.build.empty() ? "-" : "`" + st.build + "`"); + } + md += "\n"; +} + +// The builds the runner made (B4 `builds.ndjson`): the slowest edges, and the packages the cache +// served. +void builds_section(std::string& md, const std::vector& runs, int top) { + std::vector ran; + std::size_t cached_packages = 0, cached_units = 0; + double total = 0, cpu = 0; + for (const auto& r : runs) + for (const auto& b : r.builds) { + if (b.cached) { + ++cached_packages; + cached_units += std::size_t(b.units); + continue; + } + ran.push_back(&b); + total += b.ms; + cpu += b.cpu_ms.value_or(0); + } + if (ran.empty() && cached_packages == 0) + return; + std::ranges::sort(ran, std::greater{}, [](const build_edge* b) { return b->ms; }); + md += + std::format("## Builds\n\n{} edge(s) ran, {} of edge time, {} of processor time; the cache " + "served {} package(s), {} unit(s).\n\n" + "| edge | package | program | time | processor | largest resident set |\n" + "|---|---|---|---:|---:|---:|\n", + ran.size(), sec(total), sec(cpu), cached_packages, cached_units); + for (std::size_t i = 0; i < ran.size() && int(i) < top; ++i) + md += std::format("| `{}` | {} | {} | {} | {} | {} |\n", cell(ran[i]->output, 80), + ran[i]->package.empty() ? "-" : ran[i]->package, + ran[i]->program.empty() ? "-" : ran[i]->program, sec(ran[i]->ms), + ran[i]->cpu_ms ? sec(*ran[i]->cpu_ms) : "-", + ran[i]->peak_bytes ? bytes(*ran[i]->peak_bytes) : "-"); + md += "\n"; +} + +// The cases that took more than four fifths of their time limit (SPEC-017 O5): those a slower +// machine fails first, with the machine's class. +void margin_section(std::string& md, const std::vector& runs, const report_options& o) { + std::string rows; + for (const auto& r : runs) + for (const auto& c : r.cases) + if (in_scope(o, c) && c.limit_ms > 0 && c.ms > 0.8 * c.limit_ms) + rows += std::format("| `{}` | {} | {} | {:.0f}% | {} |\n", + case_label(c.program, c.id), sec(c.ms), sec(c.limit_ms), + 100 * c.ms / c.limit_ms, r.machine.empty() ? "-" : r.machine); + if (rows.empty()) + return; + md += "## Margin\n\nCases that took more than four fifths of their time limit.\n\n" + "| case | time | limit | share | machine |\n|---|---:|---:|---:|---|\n" + + rows + "\n"; +} + +// The cases over the processor budget, and each ledger's state (0.11.0). +void budget_section(std::string& md, const run_record& r, const report_options& o) { + if (!o.cfg) + return; + const auto over = budget_overruns(*o.cfg, r); + if (over.empty()) + return; + md += "## Budget\n\n| case | processor time | limit | waived |\n|---|---:|---:|---|\n"; + for (const auto& x : over) + if (!x.value("stale_waiver", false)) + md += std::format("| `{}` | {} | {} | {} |\n", x.value("case", ""), + sec(x.value("cpu_ms", 0.0)), sec(x.value("limit_ms", 0.0)), + x.value("waived", false) ? "yes" : "**no**"); + else + md += std::format("| `{}` | - | - | a waiver that excuses nothing |\n", + x.value("case", "")); + md += "\n"; +} + +void ledgers_section(std::string& md, const store& s, const run_record& r, + const report_options& o) { + if (!o.cfg || o.cfg->ledgers.empty()) + return; + md += "## Ledgers\n\n| ledger | evidence | requirements | met | waived | missing |\n" + "|---|---|---:|---:|---:|---:|\n"; + std::string missing; + for (const auto& l : ledger_table(*o.cfg, s, r)) { + if (l.contains("error")) { + md += std::format("| {} | - | - | - | - | {} |\n", l.value("ledger", ""), + l.value("error", "")); + continue; + } + md += std::format("| {} | {} | {} | {} | {} | {} |\n", l.value("ledger", ""), + l.value("evidence", ""), l.value("requirements", 0), l.value("met", 0), + l["waived"].size(), l["missing"].size()); + for (const auto& m : l["missing"]) + missing += std::format("- `{}`: {}\n", l.value("ledger", ""), m.get()); + } + md += "\n" + (missing.empty() ? "" : "Missing:\n\n" + missing + "\n"); +} + void kinds(std::string& md, const std::vector& cases) { std::map> by; for (auto* c : cases) { @@ -421,6 +541,17 @@ std::string markdown_report(const store& s, const std::vector& runs, } if (wants(o, "slowest")) slowest(md, cases, o.top); + if (wants(o, "steps")) + steps_section(md, cases, o.top); + if (wants(o, "builds")) + builds_section(md, runs, o.top); + if (wants(o, "margin")) + margin_section(md, runs, o); + if (wants(o, "budget")) + for (const auto& r : runs) + budget_section(md, r, o); + if (wants(o, "ledgers") && !runs.empty()) + ledgers_section(md, s, runs.front(), o); if (wants(o, "metrics")) metrics(md, cases, o.top); if (wants(o, "resources")) diff --git a/engine/src/report.cppm b/engine/src/report.cppm index e37d7c1..eafcb19 100644 --- a/engine/src/report.cppm +++ b/engine/src/report.cppm @@ -16,12 +16,14 @@ struct report_options { int top = 10; int max_nodes = 60; std::string title; // empty: "mtest report" + // The configuration, for the sections it judges (budget, ledgers); none: those are left out. + const config* cfg = nullptr; }; // Every section a report knows, in the order it writes them. inline constexpr std::string_view kSections[] = { - "summary", "kinds", "failures", "slowest", "metrics", "resources", - "flaky", "trend", "path", "graph", "gantt", + "summary", "kinds", "failures", "slowest", "steps", "builds", "margin", "budget", + "ledgers", "metrics", "resources", "flaky", "trend", "path", "graph", "gantt", }; std::string markdown_report(const store& s, const std::vector& runs, diff --git a/engine/src/schedule.cpp b/engine/src/schedule.cpp index 678d39d..872a6b0 100644 --- a/engine/src/schedule.cpp +++ b/engine/src/schedule.cpp @@ -406,7 +406,9 @@ run_record execute(const config& c, const store& s, const plan& p, const execute rec.started = now_iso(); rec.system = std::string(os::name(os::host())); rec.arch = std::string(os::name(os::ARCH)); - rec.machine = os::env::get("HOSTNAME").value_or(os::env::get("COMPUTERNAME").value_or("")); + rec.machine = machine_class(); + rec.processors = int(os::machine::processors().logical); + rec.memory = os::machine::memory().value_or(0); rec.commit = git_commit(c.root); rec.set = o.set; rec.selection = o.selection; @@ -548,7 +550,47 @@ run_record execute(const config& c, const store& s, const plan& p, const execute {"total", items.size()}, {"started", rec.started}}); + // ONE BUDGET FOR THE MACHINE (SPEC-017 O11, 0.11.0): a directory of slot files, one per job; + // each worker holds one while it runs programs, and states it to them, so that the parallel + // work a program starts (a build's compilers) takes the slots no worker holds and the machine + // runs no more than its jobs at once. Inside another runner's budget, that one is used: the + // first worker has the slot the runner holds for this engine, the others wait for free ones. + const auto outer_budget = os::env::get("MTEST_BUDGET"); + const auto outer_held = os::env::get("MTEST_BUDGET_HELD"); + const bool nested = outer_budget && !outer_budget->empty(); + const auto budget_dir = + nested ? os::fs::from_utf8(*outer_budget) : s.dir / "work" / rec.id / "budget"; + if (!nested) { + std::filesystem::create_directories(budget_dir); + for (int i = 0; i < rec.jobs; ++i) + (void)os::fs::write(budget_dir / std::format("slot-{}", i), ""); + } + rec.budget = nested ? 0 : rec.jobs; + // A free slot: the first that locks without waiting, else the worker's own number, waited for. + auto take_slot = [&](int worker) -> std::pair { + if (nested && worker == 0 && outer_held && !outer_held->empty()) + return {os::fs::from_utf8(*outer_held), os::fs::lock{}}; + std::vector slots; + for (const auto& e : std::filesystem::directory_iterator(budget_dir)) + if (e.path().filename().string().starts_with("slot-")) + slots.push_back(e.path()); + std::ranges::sort(slots); + for (const auto& f : slots) + if (outer_held.value_or("") != os::fs::utf8(f)) + if (auto l = os::fs::lock::acquire(f, os::fs::lock::kind::exclusive, false); + l && l->held()) + return {f, std::move(*l)}; + if (slots.empty()) + return {{}, os::fs::lock{}}; + const auto f = slots[std::size_t(worker) % slots.size()]; + auto l = os::fs::lock::acquire(f, os::fs::lock::kind::exclusive, true); + return {f, l ? std::move(*l) : os::fs::lock{}}; + }; + rec.jitter_ms = o.jitter_ms; + rec.jitter_seed = o.jitter_ms > 0 ? o.jitter_seed.value_or(std::random_device{}()) : 0; + auto worker = [&](int slot) { + auto [slot_file, slot_lock] = take_slot(slot); for (;;) { std::size_t mine = 0; { @@ -587,6 +629,14 @@ run_record execute(const config& c, const store& s, const plan& p, const execute req.select = it.select; req.seed = o.seed; req.env = o.env; + if (!slot_file.empty()) { + req.env.emplace_back("MTEST_BUDGET", os::fs::utf8(budget_dir)); + req.env.emplace_back("MTEST_BUDGET_HELD", os::fs::utf8(slot_file)); + } + if (rec.jitter_ms > 0) { + req.env.emplace_back("MTEST_JITTER", std::to_string(rec.jitter_ms)); + req.env.emplace_back("MTEST_JITTER_SEED", std::to_string(rec.jitter_seed + mine)); + } req.run_under = o.run_under; req.monitor = rec.monitor; req.sample_ms = c.monitor.sample_ms; @@ -702,6 +752,10 @@ run_record execute(const config& c, const store& s, const plan& p, const execute cr.only = ci->info.only; cr.except = ci->info.except; cr.bounds = ci->info.bounds; + if (ci->info.timeout > 0) + cr.limit_ms = ci->info.timeout * 1000; + else if (info->program.timeout.count() > 0) + cr.limit_ms = double(info->program.timeout.count()) * 1000; judge_bounds(cr, ci->info.stated_bounds); expected_to_fail(cr, ci->info.expect_fail); } else if (cr.kind.empty()) { diff --git a/engine/src/schedule.cppm b/engine/src/schedule.cppm index d371c8a..67693b3 100644 --- a/engine/src/schedule.cppm +++ b/engine/src/schedule.cppm @@ -25,6 +25,10 @@ struct execute_options { // `--coverage`: each program writes its `.gcda` files below a directory of its own here // (`GCOV_PREFIX`), and `coverage_cases` receives the cases each directory serves. std::optional coverage_dir; + // O12 (0.11.0): the most a program waits at a contention point, and the seed of the waits; + // zero: none. A run with jitter is a search for an order defect, never a verdict to merge by. + int jitter_ms = 0; + std::optional jitter_seed; int coverage_strip = 0; std::map>* coverage_cases = nullptr; }; diff --git a/engine/src/select.cpp b/engine/src/select.cpp index fd311cc..6abf0e8 100644 --- a/engine/src/select.cpp +++ b/engine/src/select.cpp @@ -461,8 +461,13 @@ struct history { std::set failed_recently; // within regression_days }; +// A duration measured on a machine of this one's class is preferred to one measured on another: +// hosted runners of one label are processors that differ by up to 1.6 times (P12.4, 0.11.0). A +// case measured only elsewhere keeps that measurement. history read_history(const config& c, const store& s) { history h; + std::map elsewhere; + const auto here = machine_class(); const auto all = runs(s); const auto cutoff = std::format("{:%Y%m%d}", @@ -478,10 +483,12 @@ history read_history(const config& c, const store& s) { for (const auto& pr : r->programs) if (pr.usage && pr.usage->peak_bytes && !h.peak.contains(pr.name)) h.peak[pr.name] = *pr.usage->peak_bytes; + const bool same = r->machine.empty() || r->machine == here; for (const auto& cr : r->cases) { const auto key = cr.program + "::" + cr.id; - if (!h.ms.contains(key)) - h.ms[key] = cr.ms; + auto& into = same ? h.ms : elsewhere; + if (!into.contains(key)) + into[key] = cr.ms; if (failing(cr.status)) { h.failed_recently.insert(key); if (last) @@ -489,6 +496,8 @@ history read_history(const config& c, const store& s) { } } } + for (auto& [key, ms] : elsewhere) + h.ms.try_emplace(key, ms); return h; } diff --git a/engine/src/service.cpp b/engine/src/service.cpp index 11d1feb..d3784d7 100644 --- a/engine/src/service.cpp +++ b/engine/src/service.cpp @@ -56,7 +56,7 @@ int serve(const std::filesystem::path& cwd) { {"result", {{"protocolVersion", params.value("protocolVersion", std::string("2025-06-18"))}, {"capabilities", {{"tools", {{"listChanged", false}}}}}, - {"serverInfo", {{"name", "mtest"}, {"version", "0.10.0"}}}, + {"serverInfo", {{"name", "mtest"}, {"version", "0.11.0"}}}, {"instructions", "Run tests and ask about runs: run, plan, explain, history, analyze, graph, " "gate, ci. A test session (test/plan, test/run, test/add, test/cancel, " diff --git a/engine/src/store.cpp b/engine/src/store.cpp index 08e6a44..9996b69 100644 --- a/engine/src/store.cpp +++ b/engine/src/store.cpp @@ -123,6 +123,8 @@ std::optional load_run(const store& s, std::string_view id) { r.programs.push_back(program_result_from(j)); for (const auto& j : read_lines(dir / "fixtures.ndjson")) r.fixtures.push_back(fixture_run_from(j)); + for (const auto& j : read_lines(dir / "builds.ndjson")) + r.builds.push_back(build_edge_from(j)); return r; } diff --git a/engine/tests/engine.cpp b/engine/tests/engine.cpp index 5d5f53a..348a66c 100644 --- a/engine/tests/engine.cpp +++ b/engine/tests/engine.cpp @@ -266,6 +266,95 @@ int main() { expect(!over["pass"].get()); expect(contains(over["reasons"][0].get(), "fail")); }; + "a case over its processor budget fails the gate unless a waiver names it"_test[kind:: + e2e] = [] { + const std::string slow = + script({"slow:spend", "light:pass"}, "spend() { mtest_metric cpu.user 5000 ms; }\n"); + auto root = project( + {{"mtest.toml", + "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n[budget]\ncase_cpu = \"1s\"\n"}, + {"b.sh", slow}}); + (void)mtest(root, {"run"}); + auto over = json_of(mtest(root, {"gate", "--set", "full", "--json"})); + expect(!over["pass"].get()) << over.dump(); + expect(contains(over.dump(), "over the case budget")) << over.dump(); + (void)mcpplibs::os::fs:: + write(root / "mtest.toml", + "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n[budget]\ncase_cpu = \"1s\"\n" + "[[budget.waiver]]\ncase = \"b::slow\"\nreason = \"it builds a toolchain\"\n"); + auto waived = json_of(mtest(root, {"gate", "--set", "full", "--json"})); + expect(waived["pass"].get()) << waived.dump(); + (void)mcpplibs::os::fs:: + write(root / "mtest.toml", + "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n[budget]\ncase_cpu = \"10s\"\n" + "[[budget.waiver]]\ncase = \"b::slow\"\nreason = \"it builds a toolchain\"\n"); + auto stale = json_of(mtest(root, {"gate", "--set", "full", "--json"})); + expect(!stale["pass"].get()) << stale.dump(); + expect(contains(stale.dump(), "excuses nothing")) << stale.dump(); + }; + "a ledger's requirement is met by a passing case's clause or excused by a waiver"_test + [kind::e2e] = [] { + auto root = project({{"mtest.toml", "[programs.b]\ncommand = [\"bash\", " + "\"b.sh\"]\n[ledger.spec]\nfile = \"reqs.txt\"\n"}, + {"reqs.txt", "# the clauses\nR1\nR2\n"}, + {"b.sh", script({"a:ok --clause R1"})}}); + (void)mtest(root, {"run"}); + auto missing = json_of(mtest(root, {"gate", "--set", "full", "--json"})); + expect(!missing["pass"].get()) << missing.dump(); + expect(contains(missing.dump(), "R2 has no passing case")) << missing.dump(); + expect(!contains(missing.dump(), "R1 has no").held) << missing.dump(); + (void)mcpplibs::os::fs:: + write(root / "mtest.toml", + "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n[ledger.spec]\nfile = " + "\"reqs.txt\"\n" + "[[ledger.spec.waiver]]\nitem = \"R2\"\nreason = \"not implemented yet\"\n"); + auto excused = json_of(mtest(root, {"gate", "--set", "full", "--json"})); + expect(excused["pass"].get()) << excused.dump(); + }; + "a program is told the run's budget and the slot held for it"_test[kind::e2e] = [] { + auto root = project( + {{"mtest.toml", "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n"}, + {"b.sh", script({"budget:told"}, + "told() { [ -d \"$MTEST_BUDGET\" ] && [ -f \"$MTEST_BUDGET_HELD\" ] " + "|| mtest_eq told untold; }\n")}}); + auto r = mtest(root, {"run", "--jobs", "2"}); + expect(e2e::succeeded(r)) << r.out << r.err; + std::optional run; + for (const auto& e : std::filesystem::recursive_directory_iterator(root / ".mtest")) + if (e.path().filename() == "run.json") + run = json::parse(*mcpplibs::os::fs::read(e.path()), nullptr, false); + expect(fatal(run.has_value())); + expect(eq(run->value("budget", 0), 2)) << run->dump(); + }; + "jitter is stated to the programs and recorded with its seed"_test[kind::e2e] = [] { + auto root = project( + {{"mtest.toml", "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n"}, + {"b.sh", script({"j:told"}, + "told() { [ \"$MTEST_JITTER\" = 5 ] && [ -n \"$MTEST_JITTER_SEED\" ] " + "|| mtest_eq told untold; }\n")}}); + auto r = mtest(root, {"run", "--jitter", "5", "--jitter-seed", "7"}); + expect(e2e::succeeded(r)) << r.out << r.err; + std::optional run; + for (const auto& e : std::filesystem::recursive_directory_iterator(root / ".mtest")) + if (e.path().filename() == "run.json") + run = json::parse(*mcpplibs::os::fs::read(e.path()), nullptr, false); + expect(fatal(run.has_value())); + expect(eq((*run)["jitter"].value("max_ms", 0), 5)) << run->dump(); + expect(eq((*run)["jitter"].value("seed", std::uint64_t{0}), std::uint64_t{7})) + << run->dump(); + }; + "a run records the machine's class, its processors and memory"_test[kind::e2e] = [] { + auto root = project({{"mtest.toml", "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\n"}, + {"b.sh", script({"a:pass"})}}); + (void)mtest(root, {"run"}); + std::optional run; + for (const auto& e : std::filesystem::recursive_directory_iterator(root / ".mtest")) + if (e.path().filename() == "run.json") + run = json::parse(*mcpplibs::os::fs::read(e.path()), nullptr, false); + expect(fatal(run.has_value())); + expect(!run->value("machine", "").empty()) << run->dump(); + expect(gt(run->value("processors", 0), 0)) << run->dump(); + }; "the graph names programs, cases, resources and artifacts"_test[kind::e2e] = [] { auto root = project({{"t.sh", "#!/usr/bin/env bash\n# mtest-protocol=1\n. '" + adapter() + "'\nok() { :; }\nmtest_case db ok --resource serial:db " @@ -520,6 +609,11 @@ printf ')" + header + expect(eq(run["status"].get(), std::string("pass"))) << run.dump(); auto map = json_of(mtest(root, {"coverage", "--json"})); expect(eq(map["cases"]["t::t"], json::array({"a.c"}))) << map.dump(); + // The function whose arcs ran, by its ident (0.11.0), and how widely the cases share it. + expect(eq(map["functions"]["t::t"], json::array({"a.c#1"}))) << map.dump(); + auto spread = json_of(mtest(root, {"coverage", "spread", "--json"})); + expect(eq(spread["functions"].get(), 1)) << spread.dump(); + expect(near(spread["median_function_share"].get(), 1.0, 1e-9)) << spread.dump(); }; "a run under --coverage maps each case to the sources it executed"_test[kind::e2e] = [] { // GCC's own `.gcda` files, where a plain `gcc` builds a program here. @@ -771,6 +865,30 @@ printf ')" + header + expect(contains(g.out, "flowchart LR")); expect(contains(g.out, "class n0,n1 crit")) << g.out; }; + "a report gives the builds a host recorded and the cases near their limit"_test[kind:: + e2e] = [] { + auto root = project( + // The case takes 85% of its limit; the rest is for a slow host's shell to start. + {{"mtest.toml", "[programs.b]\ncommand = [\"bash\", \"b.sh\"]\ntimeout = \"4s\"\n"}, + {"b.sh", script({"near:nap"}, "nap() { sleep 3.4; }\n")}}); + (void)mtest(root, {"run"}); + std::filesystem::path dir; + for (const auto& e : std::filesystem::recursive_directory_iterator(root / ".mtest")) + if (e.path().filename() == "run.json") + dir = e.path().parent_path(); + expect(fatal(!dir.empty())); + (void)mcpplibs::os::fs:: + write(dir / "builds.ndjson", + "{\"build\":\"b1\",\"output\":\"obj/big.o\",\"package\":\"app\",\"started_ms\":0," + "\"ms\":4000,\"cpu_ms\":3900}\n" + "{\"build\":\"b1\",\"package\":\"dep\",\"cached\":true,\"units\":12}\n"); + auto r = mtest(root, {"report", "--format", "md", "--sections", "builds,margin"}); + expect(contains(r.out, "## Builds")) << r.out; + expect(contains(r.out, "obj/big.o")) << r.out; + expect(contains(r.out, "12 unit(s)")) << r.out; + expect(contains(r.out, "## Margin")) << r.out; + expect(contains(r.out, "b::near")) << r.out; + }; "a markdown report has the sections asked for, with mermaid"_test[kind::e2e] = [] { auto root = project({{"a.sh", "exit 0\n"}, {"b.sh", "echo broken >&2; exit 3\n"}}); (void)mtest(root, {"run", "--json", "bash a.sh", "bash b.sh"}); diff --git a/lib/docs/writing-tests.md b/lib/docs/writing-tests.md index 8697a1f..26cc03d 100644 --- a/lib/docs/writing-tests.md +++ b/lib/docs/writing-tests.md @@ -53,7 +53,7 @@ E2E cli{"cli", resource{"serial:home"}, timeout{std::chrono::minutes{2}}, [] { / The entry enters the child before any case; the copy runs no case and reports nothing. The files of one directory are then one program (mcpp `[test] programs = "directory"`) however many of them start copies. -- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.10.0"`. +- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.11.0"`. - Identifiers are stable: a selection, a history and a shard name a case by them. ## Expectations @@ -182,6 +182,20 @@ expect(e2e::contains(r.err, "warning")); the cut. A case that looks for text in a long output raises the limit, or checks the flag first. - `e2e::tree({{path, content}...})` writes a directory of files; `e2e::golden(file, text)` compares with a golden file (`MTEST_GOLDEN=update` writes it). +- `e2e::golden(file, text, rules)` applies redactions first (`e2e::redactions::paths(root)`, + `durations()`, `sizes()`, `digests()`, or `{pattern, with}` of one's own), so a contract's whole + output is compared without what differs from run to run; an update prints what it changed by line. +- **A run is a step of the case** (P5.6): its name (`command::step`, else its words), status, time + and the processor time of the programs it ran are in the case's record. A kit that runs its + commands another way writes `step({...})`; `exercise({"command build", ...})` states the items of + the product's surface a case used, which a ledger reads (`[ledger.*]`, evidence `exercised`). +- **The end of a case checks what its programs did** (0.11.0): a process a program left running, + holding an output open, fails the case; so does a line of their output that matches a pattern the + project declares diagnostic (`e2e::diagnostics({R"(^warning:)"})`) and no pattern the case + declares (`e2e::expect_diagnostic(R"(^warning: x)")`), or a declared pattern no line matched. A + project migrating to the check runs under `MTEST_DIAGNOSTICS=report` (or + `e2e::diagnostics_policy(e2e::diagnostic_policy::report)`): the lines are printed and counted + (`diagnostics.undeclared`), and nothing fails. - **A session** drives a program whose input depends on what it wrote: a serial console, a debugger, a login, a read-eval-print loop. @@ -258,6 +272,13 @@ E2E fetch{"fetch", uses{"index"}, resource{"port"}, [] { most that many cases at once. - What a case shares only with the cases of its own program is a `shared` (above). +## Contention points + +`contention_point()` marks a place where the order of threads or processes is not fixed. Under a +runner's jitter (`mtest run --jitter MS`, SPEC-017 O12) the program waits there a random time up to +MS, the same for the same seed; elsewhere nothing happens. A case begins at one, and a shared object +is made at one. + ## Use of the machine Under a runner each case reports its processor time (`cpu.user`, `cpu.sys`), and the program its own diff --git a/lib/mcpp.toml b/lib/mcpp.toml index 2a939ec..29310cb 100644 --- a/lib/mcpp.toml +++ b/lib/mcpp.toml @@ -17,7 +17,7 @@ [package] namespace = "mcpplibs" name = "mtest" -version = "0.10.0" +version = "0.11.0" description = "The C++ test library of mtest: cases, expectations, fixtures, bench, conformance and end-to-end helpers, with no macro and no header, speaking the mtest protocol" license = "Apache-2.0" authors = ["mcpp-community"] @@ -63,5 +63,5 @@ test-session = ["mtest", "serve"] # A range, not one release: the library is in the test graph of every package # that uses it, mcpplibs.os among them, whose tests build it against the os under # test (mcpp 2026.11.9.1), and a package pins its own os. -mcpplibs.os = "^0.4.1" +mcpplibs.os = "^0.4.7" nlohmann.json = "3.12.0" diff --git a/lib/src/e2e.cpp b/lib/src/e2e.cpp index d550900..6811905 100644 --- a/lib/src/e2e.cpp +++ b/lib/src/e2e.cpp @@ -12,6 +12,132 @@ constexpr std::string_view protocol_variables[] = {"MTEST_PROTOCOL", "MTEST_MOD "MTEST_REPORT", "MTEST_SELECT", "MTEST_ISOLATION", "MTEST_TMPDIR", "MTEST_SEED"}; + +// WHAT THE PROGRAMS OF A CASE DID THAT ITS EXPECTATIONS DO NOT SEE (0.11.0), judged at the case's +// end: a process that outlived the program that started it, holding an output open, and the +// lines of their output that match a diagnostic pattern of the project. +struct watch { + std::vector outlived; // the commands whose process outlived them + std::vector lines; // diagnostic lines, as written + std::vector declared; // the patterns the case declared +}; +std::mutex watch_lock; +std::map watches; // by case +std::vector project_patterns; +std::optional policy; + +diagnostic_policy current_policy() { + if (policy) + return *policy; + return os::env::get("MTEST_DIAGNOSTICS").value_or("") == "report" ? diagnostic_policy::report + : diagnostic_policy::enforce; +} + +bool matches_any(const std::string& line, const std::vector& patterns) { + for (const auto& p : patterns) + if (std::regex_search(line, std::regex(p))) + return true; + return false; +} + +void judge(const std::string& id) { + watch w; + { + std::lock_guard g(watch_lock); + auto it = watches.find(id); + if (it == watches.end()) + return; + w = std::move(it->second); + watches.erase(it); + } + for (const auto& line : w.outlived) { + comparison c; + c.expression = "a process the program started outlived it, holding an output open"; + c.actual = line; + expect(c) << "the case's programs leave nothing running"; + } + std::vector undeclared, unseen; + for (const auto& line : w.lines) + if (!matches_any(line, w.declared)) + undeclared.push_back(line); + for (const auto& pattern : w.declared) { + bool seen = false; + for (const auto& line : w.lines) + seen = seen || std::regex_search(line, std::regex(pattern)); + if (!seen) + unseen.push_back(pattern); + } + if (current_policy() == diagnostic_policy::report) { + metric("diagnostics.undeclared", static_cast(undeclared.size()), "1"); + for (const auto& line : undeclared) + std::println(std::cerr, "mtest: {}: an undeclared diagnostic line: {}", id, line); + return; + } + for (const auto& line : undeclared) { + comparison c; + c.expression = "a diagnostic line the case did not declare (e2e::expect_diagnostic)"; + c.actual = line; + expect(c) << "every diagnostic line is declared"; + } + for (const auto& pattern : unseen) { + comparison c; + c.expression = "a diagnostic the case declared did not appear"; + c.expected = pattern; + expect(c) << "every declared diagnostic appears"; + } +} + +watch& watch_of_case(const std::string& id) { + at_case_end("mcpplibs.mtest.e2e", [id] { judge(id); }); + return watches[id]; +} + +void observe(const outcome& r) { + const auto id = current_case(); + if (id.empty()) + return; + std::vector found; + std::vector patterns; + { + std::lock_guard g(watch_lock); + patterns = project_patterns; + } + if (!patterns.empty()) + for (const auto* text : {&r.out, &r.err}) + for (auto part : std::views::split(*text, '\n')) { + std::string line(part.begin(), part.end()); + if (!line.empty() && line.back() == '\r') + line.pop_back(); + if (!line.empty() && matches_any(line, patterns)) + found.push_back(std::move(line)); + } + if (found.empty() && !r.held_open) + return; + std::lock_guard g(watch_lock); + auto& w = watch_of_case(id); + if (r.held_open) + w.outlived.push_back(r.line); + for (auto& line : found) + w.lines.push_back(std::move(line)); +} +} // namespace + +void diagnostics(std::vector patterns) { + std::lock_guard g(watch_lock); + project_patterns = std::move(patterns); +} + +void expect_diagnostic(std::string pattern) { + const auto id = current_case(); + if (id.empty()) + return; + std::lock_guard g(watch_lock); + watch_of_case(id).declared.push_back(std::move(pattern)); +} + +void diagnostics_policy(diagnostic_policy p) { + std::lock_guard g(watch_lock); + policy = p; } outcome run(command c) { @@ -41,7 +167,20 @@ outcome run(command c) { limits.hard = c.timeout; limits.out_limit = c.out_limit; limits.err_limit = c.err_limit; + step_record st; + st.name = + !c.step.empty() ? c.step : (r.line.size() > 200 ? r.line.substr(0, 200) + "..." : r.line); + st.began = std::chrono::steady_clock::now(); + const auto before = os::process::resources(os::process::of::children); auto done = os::process::run(std::move(o), limits, c.input); + st.took = std::chrono::steady_clock::now() - st.began; + if (const auto after = os::process::resources(os::process::of::children); before && after) + st.cpu_ms = std::chrono::duration((after->user + after->system) - + (before->user + before->system)) + .count(); + if (done) + st.status = done->status.shell_code(); + step(st); if (!done) { r.err = std::format("{} did not start: {}", c.argv.front(), done.error().message()); return r; @@ -55,6 +194,7 @@ outcome run(command c) { r.out = std::move(done->out); r.err = std::move(done->err); r.duration = done->duration; + observe(r); return r; } @@ -85,13 +225,83 @@ tree(std::initializer_list> files) return root; } +namespace { +// What a golden comparison compares: the text with each redaction applied, in order. +std::string redacted(std::string_view text, const std::vector& rules) { + std::string out(text); + for (const auto& r : rules) + out = std::regex_replace(out, std::regex(r.pattern), r.with); + return out; +} +std::vector lines_of(std::string_view text) { + std::vector out; + for (auto part : std::views::split(text, '\n')) + out.emplace_back(part.begin(), part.end()); + return out; +} +// What an update changed, by line: kept, changed in place, added and removed. +std::string update_summary(std::string_view before, std::string_view after) { + const auto a = lines_of(before), b = lines_of(after); + const std::size_t common = std::min(a.size(), b.size()); + std::size_t changed = 0; + for (std::size_t i = 0; i < common; ++i) + changed += a[i] != b[i]; + const std::size_t added = b.size() > a.size() ? b.size() - a.size() : 0; + const std::size_t removed = a.size() > b.size() ? a.size() - b.size() : 0; + return std::format("{} changed, {} added, {} removed", changed, added, removed); +} +} // namespace + +std::vector redactions::paths(const std::filesystem::path& root, std::string with) { + std::vector out; + for (const auto& spelling : {os::fs::utf8(root), os::fs::generic_utf8(root)}) { + std::string escaped; + for (char ch : spelling) { + if (std::string_view(R"(\^$.|?*+()[]{})").contains(ch)) + escaped += '\\'; + escaped += ch; + } + if (!escaped.empty()) + out.push_back({escaped, with}); + } + return out; +} +const std::vector& redactions::durations() { + static const std::vector r = { + {R"(\b[0-9]+(\.[0-9]+)?\s?(ns|us|µs|ms|s|m)\b)", "