From 19d32722ac0a0e5cea5aed3e6d7db786fd698db0 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sun, 22 Nov 2026 10:46:35 +0900 Subject: [PATCH 1/3] index: a refresh beside another process waits while an index the home reads has no packages, judged as a reader judges it (mcpp#927) --- ...system-architecture-optimization-design.md | 9 ++- CHANGELOG.md | 11 +++ docs/05-dependencies.md | 7 ++ docs/zh/05-dependencies.md | 5 ++ src/provider/xlings/xlings.cpp | 35 +++++++--- tests/unit/test_xlings.cpp | 70 +++++++++++++++++++ 6 files changed, 126 insertions(+), 11 deletions(-) diff --git a/.agents/docs/2026-11-12-ecosystem-architecture-optimization-design.md b/.agents/docs/2026-11-12-ecosystem-architecture-optimization-design.md index 23d244a43..d6946c223 100644 --- a/.agents/docs/2026-11-12-ecosystem-architecture-optimization-design.md +++ b/.agents/docs/2026-11-12-ecosystem-architecture-optimization-design.md @@ -1148,7 +1148,7 @@ warning: the configuration changed (debug 1 → lines): 7a72… is built in full |---|---|---| | P1 | `programs = "package"`:一个包一个程序,以第一个 glob 的固定目录命名,定义 `main` 的文件至多一个;按文件运行的各部分以文件在程序目录(其文件的公共目录)之下的路径命名,不同子目录中的同名文件因此不冲突(目录模式下即文件名,与此前相同);`"directory"` 留下固定目录中的文件时给出 note,不形成任何组时为 `degraded`(`--strict` 拒绝) | mcpp `test_targets.cppm`;e2e 的 C++ 用例 | | P2 的汇总 | `finished in X (plan · programs · build · run)`:plan 为总时长减去其余各段,programs 不足 5 ms 时不列 | mcpp `execute.cppm`;`tests/e2e/test/streams.cpp` | -| P3 | 拿不到刷新锁、且所声明的索引目录有缺失的进程,等待刷新锁(显示 `Waiting`)后再读;索引完整时仍跳过刷新、立即读取 | mcpp `xlings.cpp` | +| P3 | 拿不到刷新锁、且 home 读取的索引有缺失的进程,等待刷新锁(显示 `Waiting`)后再读;索引完整时仍跳过刷新、立即读取 | mcpp `xlings.cpp`;`tests/unit/test_xlings.cpp`(`XlingsIndexLock`,2026-11-22 补) | | P4 | 文档:`docs/08-testing.md` 的"测试构建的体积"一节(中英) | — | | P6 | 警告给出旧目录的大小与回收命令 | mcpp `execute.cppm` | @@ -1156,6 +1156,13 @@ warning: the configuration changed (debug 1 → lines): 7a72… is built in full 索引存在时,刷新以交换替换整棵树,读取者读到的是交换前或交换后的一棵完整的树。因此实施只关闭这一个窗口,读路径不加锁。 不支持原子交换的文件系统上,"移开、改名"之间的窗口与目录不存在是同一情形,读取者同样等待。 +**P3 的缺失判断(2026-11-22 补)。** 初版只判断 home 声明的索引,且以目录是否存在为准;读取者报告没有本地索引的依据 +却是 `/pkgs`(`index_status_for`)。两者不一致留下两个窗口:provider 自己的索引(`xim`)第一次取回时不在判断之内, +另一进程已建立而未写入的声明目录被判为存在。缺失的判断改为与读取者同一个函数(`index_present`),范围为 provider 自己的 +索引与声明的索引(`every_index_present`);索引状态、刷新标记与新鲜度的判断也改用它。单元测试在本进程中持有索引锁(flock +与 LockFileEx 锁的是打开的文件,同一进程的第二个持有者同样被拒绝),断言三种情形:索引完整时立即返回;声明的索引没有包、 +或 provider 自己的索引缺失时等待,锁释放后返回。 + **P2 的另两项不在本版。** `-p` 的预构建取快速路径、同一次调用内复用解析:二者改变 `prepare_build` 的入口契约(解析结果的生命周期 与键),在本轮之外;当前代价是 `-p` 多一次完整规划。 diff --git a/CHANGELOG.md b/CHANGELOG.md index fae16a7e1..72b221ebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ > Each `## []` section is that release's notes. Entries are written in English > from 2026.9.28.3 on; earlier entries remain as written. +## [Unreleased] + +### Fixed + +- A process that finds another refreshing the package index waits for it when an index the + home reads has no packages yet, and judges that as a reader does (`/pkgs`): the + provider's own index (`xim`) is one of them, and a directory the other process has made and + not yet filled is absent. Before, only the declared directories were judged, by existence, + so a reader beside a first fetch skipped the refresh and reported no local package index + (mcpp#927, design 2026-11-12 §18 P3). + ## [2026.11.21.1] - 2026-11-21 ### Added diff --git a/docs/05-dependencies.md b/docs/05-dependencies.md index f5594403b..ffa44e0ec 100644 --- a/docs/05-dependencies.md +++ b/docs/05-dependencies.md @@ -478,6 +478,13 @@ after any failed refresh. An install through xlings is stopped when xlings writes nothing, not even its heartbeat, for 300 seconds. Terminating mcpp terminates the xlings process it started. +Processes that share a home refresh the index once. A process that finds another +refreshing it reads the local index at once, unless an index the home reads (the +provider's own, `xim`, or one the home declares) has no packages yet; it then +waits for that refresh and reads what it fetched, instead of reporting no local +package index *(2026.11.13.1+; the provider's own index and a directory not yet +filled, @NEXT@+, mcpp#927)*. + Run any command with `-v` to see the decision for each dependency and why. ### Offline builds and mcpp fetch (2026.11.21.1+) diff --git a/docs/zh/05-dependencies.md b/docs/zh/05-dependencies.md index cbcf78882..4cb6f4626 100644 --- a/docs/zh/05-dependencies.md +++ b/docs/zh/05-dependencies.md @@ -400,6 +400,11 @@ mcpp index status # local state: state, age and revision 失败之后一样,继续使用本地索引。经由 xlings 的安装,在 xlings 连续 300 秒 没有任何输出(包括心跳)时被终止。终止 mcpp 会一并终止它启动的 xlings 进程。 +共用一个 home 的进程只刷新一次索引。发现另一个进程正在刷新的进程立即读取本地 +索引,除非 home 读取的某个索引(provider 自己的 `xim`,或 home 声明的)还没有 +任何包;这时它等待那次刷新结束,读取其取回的内容,而不是报告没有本地索引 +(2026.11.13.1 起;provider 自己的索引与尚未写入的目录,@NEXT@ 起,mcpp#927)。 + 对任意命令加 `-v`,可以看到每个依赖的判定结果与原因。 ### 离线构建与 mcpp fetch(2026.11.21.1 起) diff --git a/src/provider/xlings/xlings.cpp b/src/provider/xlings/xlings.cpp index 6c52e545d..999323818 100644 --- a/src/provider/xlings/xlings.cpp +++ b/src/provider/xlings/xlings.cpp @@ -77,6 +77,15 @@ std::filesystem::path index_pkgs_dir(const std::filesystem::path& indexDir) { return indexDir / "pkgs"; } +// AN INDEX IS PRESENT WHEN ITS PACKAGES ARE. A reader that finds none reports no local package +// index (`index_status_for`), and a refresh beside another process waits for it +// (`every_index_present`): one judgement, so that a directory another process has made and not +// yet filled is absent to both (mcpp#927). +bool index_present(const std::filesystem::path& indexDir) { + std::error_code ec; + return std::filesystem::exists(index_pkgs_dir(indexDir), ec) && !ec; +} + std::filesystem::path index_refresh_marker(const std::filesystem::path& indexDir) { return indexDir / ".mcpp-index-updated"; } @@ -119,7 +128,7 @@ bool official_index_cache_matches_package_file(const Env& env, } void mark_index_refreshed(const std::filesystem::path& indexDir) { - if (!std::filesystem::exists(index_pkgs_dir(indexDir))) return; + if (!index_present(indexDir)) return; std::error_code ec; std::filesystem::create_directories(indexDir, ec); auto marker = index_refresh_marker(indexDir); @@ -145,7 +154,7 @@ void mark_known_indexes_refreshed(const Env& env) { bool is_index_dir_fresh(const std::filesystem::path& indexDir, std::int64_t ttlSeconds) { std::error_code ec; - if (!std::filesystem::exists(index_pkgs_dir(indexDir))) return false; + if (!index_present(indexDir)) return false; auto marker = index_refresh_marker(indexDir); if (!std::filesystem::exists(marker)) return false; @@ -197,11 +206,9 @@ std::optional read_index_revision(const std::filesystem::path& inde IndexStatus index_status_for(const std::filesystem::path& indexDir, std::int64_t ttlSeconds) { - std::error_code ec; - bool present = std::filesystem::exists(index_pkgs_dir(indexDir), ec) && !ec; return IndexStatus{ .dir = indexDir, - .present = present, + .present = index_present(indexDir), .fresh = is_index_dir_fresh(indexDir, ttlSeconds), .ageSeconds = index_age_seconds(indexDir), .rev = read_index_revision(indexDir), @@ -2198,6 +2205,13 @@ int update_index(const Env& env, bool quiet) { } namespace { +// Every index a reader of this home reads, as it judges one: the provider's own and each declared. +bool every_index_present(const Env& env) { + return index_present(official_index_dir(env)) + && std::ranges::all_of(paths::declared_indices(env), + [](const auto& d) { return index_present(d.dir); }); +} + int update_index_unguarded(const Env& env, bool quiet) { // Offline is absolute: no caller gets to reach the network by going around // the decision layer. Reported as success so a build that can still resolve @@ -2226,11 +2240,12 @@ int update_index_unguarded(const Env& env, bool quiet) { // was still fetching, or exchanging on a volume without an atomic // exchange, and reported "no local package index" for a home that had // one a moment later. Such a reader waits for the refresh it skipped; - // one with an index skips as before. - bool absent = false; - for (const auto& d : paths::declared_indices(env)) - if (!std::filesystem::exists(d.dir, lockEc)) absent = true; - if (!absent) { + // one with every index skips as before. Every index is the provider's + // own (xim) and each the home declares, judged as a reader judges one + // (`index_present`): the declared directories alone, by existence, + // let a reader skip beside a first fetch of xim, and beside a declared + // directory made and not yet filled. + if (every_index_present(env)) { mcpp::log::verbose("index", "another process is refreshing the index — skipping this one"); return 0; diff --git a/tests/unit/test_xlings.cpp b/tests/unit/test_xlings.cpp index 9afc1bf52..94dcdf332 100644 --- a/tests/unit/test_xlings.cpp +++ b/tests/unit/test_xlings.cpp @@ -4,6 +4,8 @@ import mcpplibs.mtest; import mcpplibs.os.platform; import mcpp.provider.xlings; import mcpp.invocation; +import mcpp.effects; +import mcpp.home.files; using namespace mcpplibs::mtest; @@ -839,3 +841,71 @@ UTest a_projects_install_takes_the_homes_turn{"XlingsState/AProjectsInstallTakes std::filesystem::remove_all(home); std::filesystem::remove_all(project); }}; + +// ─── A refresh beside another process (mcpp#927, design 2026-11-12 §18 P3) ─── +// +// The test holds the index lock as another process would. A second holder in this process is +// refused as one in another would be (flock and LockFileEx lock an open file, not a process), so +// the refresh below never reaches a sync: it skips, or it waits for this lock. +namespace { + +// A home whose provider's index (xim) and one declared index (`mcpplibs`) are laid out as a reader +// finds them: `/pkgs`. +std::filesystem::path home_with_indices() { + auto home = make_tempdir("mcpp-xlings-index-lock"); + std::ofstream(home / ".xlings.json") + << R"({"index_repos": [{"name": "mcpplibs", "url": "https://example.invalid/index.git"}]})"; + std::filesystem::create_directories(home / "data" / "xim-pkgindex" / "pkgs"); + std::filesystem::create_directories(home / "data" / "mcpplibs" / "pkgs"); + return home; +} + +// Whether a refresh started while this process holds the home's index lock waited for it: it had +// not returned after a second and a half. `fill` then completes the home, as the other process's +// fetch would, and the lock is released; the refresh returns 0 either way. +bool refresh_waited_beside_another(const std::filesystem::path& home, const std::function& fill) { + std::future refresh; // declared first, so that it ends after the lock is released + auto held = mcpp::home::FileLock::try_acquire(home / "data"); + expect(fatal(held.has_value())) << "the index lock could not be taken"; + expect(fatal(!mcpp::home::FileLock::try_acquire(home / "data").has_value())) + << "a second holder in this process was not refused: the refresh would run a sync"; + const mcpp::provider::xlings::Env env{.home = home}; + refresh = std::async(std::launch::async, [env] { return mcpp::provider::xlings::update_index(env, true); }); + const bool waited = refresh.wait_for(std::chrono::milliseconds{1500}) == std::future_status::timeout; + fill(); + held.reset(); + expect(fatal(refresh.wait_for(std::chrono::seconds{30}) == std::future_status::ready)) + << "the refresh did not return once the lock was released"; + expect(eq(refresh.get(), 0)); + return waited; +} + +} // namespace + +UTest skips_beside_another_when_every_index_is_present{"XlingsIndexLock/SkipsBesideAnotherWhenEveryIndexIsPresent", [] { + if (mcpp::effects::offline_mode()) skip("offline: a refresh returns before the lock"); + auto home = home_with_indices(); + expect(!refresh_waited_beside_another(home, [] {})) << "a refresh waited although every index was in place"; + std::filesystem::remove_all(home); +}}; + +// The window #927 reported: a declared index's directory made by the other process's fetch and not +// yet filled is absent to a reader, so it is absent to the refresh that would skip. +UTest waits_beside_another_for_an_index_not_yet_filled{"XlingsIndexLock/WaitsBesideAnotherForAnIndexNotYetFilled", [] { + if (mcpp::effects::offline_mode()) skip("offline: a refresh returns before the lock"); + auto home = home_with_indices(); + std::filesystem::remove_all(home / "data" / "mcpplibs" / "pkgs"); + expect(refresh_waited_beside_another(home, [&] { std::filesystem::create_directories(home / "data" / "mcpplibs" / "pkgs"); })) + << "a refresh skipped beside another while a declared index had no packages"; + std::filesystem::remove_all(home); +}}; + +// The provider's own index is read as a declared one is: a first fetch of it is waited for too. +UTest waits_beside_another_for_the_providers_own_index{"XlingsIndexLock/WaitsBesideAnotherForTheProvidersOwnIndex", [] { + if (mcpp::effects::offline_mode()) skip("offline: a refresh returns before the lock"); + auto home = home_with_indices(); + std::filesystem::remove_all(home / "data" / "xim-pkgindex"); + expect(refresh_waited_beside_another(home, [&] { std::filesystem::create_directories(home / "data" / "xim-pkgindex" / "pkgs"); })) + << "a refresh skipped beside another while the provider's own index was absent"; + std::filesystem::remove_all(home); +}}; From bd9b38962eef674e55454323e8d33f262bee7bb1 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sun, 22 Nov 2026 11:08:24 +0900 Subject: [PATCH 2/3] test: a suite run as a case is counted by its cases on the program's line, as the summary counts it --- CHANGELOG.md | 3 +++ src/build/execute.cppm | 25 ++++++++++++++++--------- tests/e2e/test/protocol.cpp | 23 ++++++++++++++++++++++- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72b221ebd..799c97320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ not yet filled is absent. Before, only the declared directories were judged, by existence, so a reader beside a first fetch skipped the refresh and reported no local package index (mcpp#927, design 2026-11-12 §18 P3). +- `mcpp test` counts a suite run as a case by its cases on the program's line as the summary + does: an mtest program of one suite and seven cases read `8 cases` on its line and + `7 passed` in the summary. ## [2026.11.21.1] - 2026-11-21 diff --git a/src/build/execute.cppm b/src/build/execute.cppm index e1e3502d6..403b3a7cc 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -5486,6 +5486,16 @@ static void test_resolve_runner(TestBuild& tb, const TestOptions& testOpts, bool } } +// A CASE THAT DECLARES CASES (a suite run as a case) is told by them: neither a +// program's line nor the run's summary counts it beside them, so that the two +// counts of one run agree (an mtest program of one suite and seven cases read +// `8 cases` on its line and `7 passed` in the summary). +bool declares_cases(const mcpp::build::test_protocol::Report& rep, const std::string& id) { + return std::ranges::any_of(rep.cases, [&](const auto& c) { + return c.id.size() > id.size() && c.id.starts_with(id) && c.id[id.size()] == '/'; + }); +} + // A failed test whose file imports a module no package of this build provides: // the compiler says only that the module was not built. A test's framework is a // `[dev-dependencies]` entry (docs/08), which a manifest written over the @@ -6007,8 +6017,10 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, report = {}; spoken = 0; } else { - casesNote = std::format(", {} case{}", report.cases.size(), - report.cases.size() == 1 ? "" : "s"); + const auto told = std::ranges::count_if(report.cases, [&](const auto& c) { + return !declares_cases(report, c.id); + }); + casesNote = std::format(", {} case{}", told, told == 1 ? "" : "s"); if (exitCode == 0 && !timedOut && report.failed()) exitCode = 1; } } @@ -6858,15 +6870,10 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, // THE CASES, BESIDE THE PROGRAMS THE COUNT ABOVE IS OF. A case that // declared cases (a suite run as a case) is told by them, not counted. - auto declares = [](const mcpp::build::test_protocol::Report& rep, const std::string& id) { - return std::ranges::any_of(rep.cases, [&](const auto& c) { - return c.id.size() > id.size() && c.id.starts_with(id) && c.id[id.size()] == '/'; - }); - }; int casesPassed = 0, casesFailed = 0, casesSkipped = 0; for (auto& r : results) for (const auto& c : r.report.cases) { - if (declares(r.report, c.id)) continue; + if (declares_cases(r.report, c.id)) continue; if (c.status == "pass" || c.status == "xfail") ++casesPassed; else if (c.status == "fail" || c.status == "xpass") ++casesFailed; else if (c.status == "skip") ++casesSkipped; @@ -6905,7 +6912,7 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, continue; std::println(" {}", r.name); for (const auto& c : r.report.cases) { - if ((c.status != "fail" && c.status != "xpass") || declares(r.report, c.id)) continue; + if ((c.status != "fail" && c.status != "xpass") || declares_cases(r.report, c.id)) continue; if (c.failures.empty()) std::println(" {} {}", c.id, c.status == "xpass" ? "passed, expected to fail" : c.reason.empty() ? std::string("failed") : c.reason); diff --git a/tests/e2e/test/protocol.cpp b/tests/e2e/test/protocol.cpp index ece756f3f..542641d07 100644 --- a/tests/e2e/test/protocol.cpp +++ b/tests/e2e/test/protocol.cpp @@ -1,6 +1,6 @@ // Test programs that speak mtest's test protocol: their cases, live case records, the spec's // launch, capabilities, checks and output (formerly tests/e2e/1035, 1065 and 1070); run fixtures; -// the time the kit gives a command within its case. +// the time the kit gives a command within its case; a suite counted by its cases. import std; import mcpplibs.mtest; import mcpp.test.support; @@ -470,6 +470,27 @@ int main() {{ // the program without the case's report of a hung command: ten minutes bound it. expect(fatal(s::mcpp_command({"--version"}, s::case_dir()).timeout == std::chrono::minutes{10})); }; + + "1143 a suite is told by its cases on the program's line and in the summary"_test = [] { + // An mtest program of one suite of two cases and one case alone reports four cases, the + // suite among them; the suite is told by its cases, so the program's line and the run's + // summary both count three (an mtest program of one suite and seven cases read `8 cases` + // on its line and `7 passed` in the summary, 2026.11.21.1). + auto p = s::project::scaffold("suites"); + for (const auto& f : s::glob(p.at("tests"), "*.cpp")) fs::remove(f); + p.write("tests/cases.cpp", R"(import mcpplibs.mtest; +using namespace mcpplibs::mtest; +UTest suite{"suite", [] { + "first"_test = [] { expect(eq(1, 1)); }; + "second"_test = [] { expect(eq(2, 2)); }; +}}; +UTest alone{"alone", [] { expect(eq(3, 3)); }}; +)"); + auto r = p.mcpp({"test"}); + expect(fatal(e2e::succeeded(r))) << s::text(r); + expect(s::has_line(s::text(r), R"(cases \.\.\. ok \([0-9.]+s, 3 cases\))")) << "the program's line counts the suite: " << s::text(r); + expect(e2e::contains(s::text(r), "cases: 3 passed, 0 failed, 0 skipped")) << s::text(r); + }; }}; } // namespace From 613e8db860a5105ea279bed544fade55ebbba416 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sun, 22 Nov 2026 11:43:20 +0900 Subject: [PATCH 3/3] test: case 1038 reads the whole run of a suite as its two cases --- tests/e2e/test/affected.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test/affected.cpp b/tests/e2e/test/affected.cpp index f17680ec9..33ca927a7 100644 --- a/tests/e2e/test/affected.cpp +++ b/tests/e2e/test/affected.cpp @@ -134,7 +134,8 @@ UTest g{"g", [] { expect(s::has_line(s::text(f1), R"(test_smoke \.\.\. ok \(.*1 case\))")) << "a declared case is not selected alone"; auto f2 = grp.mcpp({"test", "--case", "g/t*"}); expect(e2e::succeeded(f2)); - expect(s::has_line(s::text(f2), R"(test_smoke \.\.\. ok \(.*3 cases\))")) << "a pattern below the listed case does not run it whole"; + // Run whole: both of its cases (the suite is told by them, as the summary counts it). + expect(s::has_line(s::text(f2), R"(test_smoke \.\.\. ok \(.*2 cases\))")) << "a pattern below the listed case does not run it whole"; }; "1120 a header affects the tests built from the units that read it"_test[s::gcc] = [] {