From f409f95480f9a79fb9bd285e6c1ff351ff89ddfa Mon Sep 17 00:00:00 2001 From: justrach <54503978+justrach@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:09:45 +0900 Subject: [PATCH] release: 0.2.5862 enable bounded bbtree mmap dictionaries Warm word lookups currently probe variable-length mmap records with binary search. Port the upstream inline-key read path to Zig and enable it by default without changing postings, ranking, hosted fallback, or candidate breadth. Keep all pages in one exact-size slab capped at 8 MiB per word index. Retain complete binary search if the budget or allocation fails, and free the slab before promotion. Native CLI peak RSS rises by 0.47-1.39 MiB (under 5%) on the three measured corpora; lookup parity and all 76 MCP scenarios pass. Build on current main, which already includes the isolated watcher CPU fixes from #758/#771, rather than merging unrelated changes from #750. Include the source attribution, reproducible benchmarks, aggregate resource evidence, regressions, release notes, and synchronized native/npm version metadata. Fixes #772 Co-Authored-By: Codegraff --- CHANGELOG.md | 6 + build.zig | 16 + build.zig.zon | 2 +- docs/measurements/bbtree-0.2.5862.json | 469 ++++++++++++++++++ docs/releases/0.2.5862.md | 78 +++ experiments/bbtree/LICENSE-upstream | 230 +++++++++ experiments/bbtree/README.md | 109 ++++ experiments/bbtree/codedb-warm-results.txt | 16 + experiments/bbtree/codegraff-warm-results.txt | 16 + experiments/bbtree/cross-corpus-report.md | 113 +++++ experiments/bbtree/cross-corpus-summary.json | 71 +++ experiments/bbtree/rss.py | 101 ++++ experiments/bbtree/run.py | 122 +++++ experiments/bbtree/warm-results.txt | 16 + experiments/bbtree/zigrepper-warm-results.txt | 16 + npm/package.json | 2 +- src/bbtree.LICENSE | 21 + src/bbtree_dictionary.zig | 293 +++++++++++ src/bench_bbtree.zig | 117 +++++ src/index.zig | 42 +- src/release_info.zig | 2 +- src/test_bbtree.zig | 73 +++ src/test_bbtree_memory.zig | 26 + src/test_index.zig | 6 + 24 files changed, 1955 insertions(+), 8 deletions(-) create mode 100644 docs/measurements/bbtree-0.2.5862.json create mode 100644 docs/releases/0.2.5862.md create mode 100644 experiments/bbtree/LICENSE-upstream create mode 100644 experiments/bbtree/README.md create mode 100644 experiments/bbtree/codedb-warm-results.txt create mode 100644 experiments/bbtree/codegraff-warm-results.txt create mode 100644 experiments/bbtree/cross-corpus-report.md create mode 100644 experiments/bbtree/cross-corpus-summary.json create mode 100644 experiments/bbtree/rss.py create mode 100644 experiments/bbtree/run.py create mode 100644 experiments/bbtree/warm-results.txt create mode 100644 experiments/bbtree/zigrepper-warm-results.txt create mode 100644 src/bbtree.LICENSE create mode 100644 src/bbtree_dictionary.zig create mode 100644 src/bench_bbtree.zig create mode 100644 src/test_bbtree.zig create mode 100644 src/test_bbtree_memory.zig diff --git a/CHANGELOG.md b/CHANGELOG.md index c387ec0d..2a09729d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +## 0.2.5862 - 2026-10-27 + +- Enable the Zig bbtree inline-key dictionary by default for warm mmap word indexes. Keep postings zero-copy and preserve exact/prefix results, ranking, hosted fallback, and ANN candidate breadth (#772). +- Bound accelerator memory to one exact-size slab, at most 8 MiB per mmap word index. Over-budget dictionaries and allocation failures retain complete binary search; promotion frees the slab before unmapping. +- Include the isolated macOS watcher CPU fixes already integrated in 0.2.5861 via #758/#771, without importing unrelated changes from #750. + ## 0.2.5861 - 2026-10-26 - Use recursive macOS events on local APFS/HFS volumes, with at most 64 content probes every two seconds and directory audits every 60 seconds, reducing idle watcher CPU (#756, #758). Fallback filesystems retain conservative polling. Each MCP client still owns a watcher. diff --git a/build.zig b/build.zig index 9cfce65f..dddb19c3 100644 --- a/build.zig +++ b/build.zig @@ -84,6 +84,20 @@ pub fn build(b: *std.Build) void { const run_step = b.step("run", "Run codedb daemon"); run_step.dependOn(&run_cmd.step); + // Opt-in, local-only warm mmap dictionary A/B experiment. + const bb_bench = b.addExecutable(.{ + .name = "bench-bbtree", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/bench_bbtree.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }), + }); + const bb_run = b.addRunArtifact(bb_bench); + bb_run.addPassthruArgs(); + b.step("bench-bbtree", "Compare bbtree and binary search on warm mmap words").dependOn(&bb_run.step); + // ── Tests (split into independent binaries for faster compilation) ── const test_filter = b.option([]const u8, "test-filter", "Only run tests whose name contains this substring"); const test_step = b.step("test", "Run all tests"); @@ -92,6 +106,8 @@ pub fn build(b: *std.Build) void { .{ .name = "test-core", .path = "src/test_core.zig", .needs_nanoregex = false }, .{ .name = "test-explore", .path = "src/test_explore.zig", .needs_nanoregex = true }, .{ .name = "test-index", .path = "src/test_index.zig", .needs_nanoregex = true }, + .{ .name = "test-bbtree", .path = "src/test_bbtree.zig", .needs_nanoregex = false }, + .{ .name = "test-bbtree-memory", .path = "src/test_bbtree_memory.zig", .needs_nanoregex = false }, .{ .name = "test-watcher", .path = "src/watcher.zig", .needs_nanoregex = true }, .{ .name = "test-parser", .path = "src/test_parser.zig", .needs_nanoregex = true }, .{ .name = "test-search", .path = "src/test_search.zig", .needs_nanoregex = true }, diff --git a/build.zig.zon b/build.zig.zon index 82f8484a..4d2aa912 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .codedb2, .fingerprint = 0x6e18d96ca2a31757, - .version = "0.2.5861", + .version = "0.2.5862", .minimum_zig_version = "0.17.0-dev.813+2153f8143", .dependencies = .{ .nanoregex = .{ diff --git a/docs/measurements/bbtree-0.2.5862.json b/docs/measurements/bbtree-0.2.5862.json new file mode 100644 index 00000000..3ee79ebf --- /dev/null +++ b/docs/measurements/bbtree-0.2.5862.json @@ -0,0 +1,469 @@ +{ + "version": "0.2.5862", + "method": { + "platform": "arm64 macOS", + "zig": "0.17.0-dev.813+2153f8143", + "lookup_rounds": 7, + "rss_pairs": 6, + "rss_command": "bench-engine word function 10000", + "rss_metric": "macOS time -l peak resident bytes", + "accelerator_cap_bytes": 8388608 + }, + "corpora": { + "codedb": { + "lookup": { + "corpus": { + "files": 64, + "source_bytes": 3123603, + "words": 13031, + "queries": 39093, + "prefix_parity": 1000, + "extra_bytes": 445440, + "build_ms": 0.127 + }, + "exact": { + "binary_median_ns": 129.96, + "bbtree_median_ns": 90.94, + "paired_latency_change_pct": -29.572649572649567 + }, + "prefix": { + "binary_median_ns": 469.0, + "bbtree_median_ns": 389.0, + "paired_latency_change_pct": -18.76332622601279 + } + }, + "native_cli_rss": { + "source_files": 64, + "skipped_source_files": 0, + "binary_median_peak_rss_bytes": 10158080.0, + "bbtree_median_peak_rss_bytes": 10649600.0, + "median_peak_rss_delta_bytes": 491520.0, + "median_peak_rss_delta_pct": 4.838709677419355, + "samples": [ + { + "implementation": "binary", + "peak_rss_bytes": 10174464, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 0 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 10633216, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 0 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 10616832, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 1 + }, + { + "implementation": "binary", + "peak_rss_bytes": 10158080, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 1 + }, + { + "implementation": "binary", + "peak_rss_bytes": 10158080, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 2 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 10731520, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 2 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 10682368, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 3 + }, + { + "implementation": "binary", + "peak_rss_bytes": 10141696, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 3 + }, + { + "implementation": "binary", + "peak_rss_bytes": 10174464, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 4 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 10665984, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 4 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 10600448, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 5 + }, + { + "implementation": "binary", + "peak_rss_bytes": 10158080, + "hits": 247, + "p50_ns": 0, + "p99_ns": 1000, + "pair": 5 + } + ] + } + }, + "codegraff": { + "lookup": { + "corpus": { + "files": 2119, + "source_bytes": 16982003, + "words": 42100, + "queries": 126300, + "prefix_parity": 1000, + "extra_bytes": 1436672, + "build_ms": 0.568 + }, + "exact": { + "binary_median_ns": 289.94, + "bbtree_median_ns": 166.0, + "paired_latency_change_pct": -44.70285175241274 + }, + "prefix": { + "binary_median_ns": 581.0, + "bbtree_median_ns": 489.0, + "paired_latency_change_pct": -13.237639553429027 + } + }, + "native_cli_rss": { + "source_files": 2119, + "skipped_source_files": 0, + "binary_median_peak_rss_bytes": 47079424.0, + "bbtree_median_peak_rss_bytes": 48537600.0, + "median_peak_rss_delta_bytes": 1458176.0, + "median_peak_rss_delta_pct": 3.097268139899078, + "samples": [ + { + "implementation": "binary", + "peak_rss_bytes": 47562752, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 0 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 48791552, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 0 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 48447488, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 1 + }, + { + "implementation": "binary", + "peak_rss_bytes": 47071232, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 3000, + "pair": 1 + }, + { + "implementation": "binary", + "peak_rss_bytes": 46923776, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 2 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 48889856, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 2 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 48463872, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 3000, + "pair": 3 + }, + { + "implementation": "binary", + "peak_rss_bytes": 47087616, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 3 + }, + { + "implementation": "binary", + "peak_rss_bytes": 46759936, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 4 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 48578560, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 4 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 48496640, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 5 + }, + { + "implementation": "binary", + "peak_rss_bytes": 47136768, + "hits": 2675, + "p50_ns": 2000, + "p99_ns": 2000, + "pair": 5 + } + ] + } + }, + "zigrepper": { + "lookup": { + "corpus": { + "files": 1548, + "source_bytes": 12160841, + "words": 36994, + "queries": 110982, + "prefix_parity": 1000, + "extra_bytes": 1262592, + "build_ms": 0.414 + }, + "exact": { + "binary_median_ns": 239.18, + "bbtree_median_ns": 165.55, + "paired_latency_change_pct": -30.244806809084402 + }, + "prefix": { + "binary_median_ns": 595.0, + "bbtree_median_ns": 533.0, + "paired_latency_change_pct": -11.472602739726023 + } + }, + "native_cli_rss": { + "source_files": 1548, + "skipped_source_files": 10, + "binary_median_peak_rss_bytes": 34430976.0, + "bbtree_median_peak_rss_bytes": 35700736.0, + "median_peak_rss_delta_bytes": 1269760.0, + "median_peak_rss_delta_pct": 3.6878420176064717, + "samples": [ + { + "implementation": "binary", + "peak_rss_bytes": 34324480, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 0 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 35700736, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 0 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 35700736, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 1 + }, + { + "implementation": "binary", + "peak_rss_bytes": 34373632, + "hits": 4214, + "p50_ns": 2000, + "p99_ns": 3000, + "pair": 1 + }, + { + "implementation": "binary", + "peak_rss_bytes": 34521088, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 2 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 35766272, + "hits": 4214, + "p50_ns": 2000, + "p99_ns": 3000, + "pair": 2 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 35651584, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 3 + }, + { + "implementation": "binary", + "peak_rss_bytes": 34308096, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 3 + }, + { + "implementation": "binary", + "peak_rss_bytes": 34537472, + "hits": 4214, + "p50_ns": 2000, + "p99_ns": 3000, + "pair": 4 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 35667968, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 4 + }, + { + "implementation": "bbtree", + "peak_rss_bytes": 36044800, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 5 + }, + { + "implementation": "binary", + "peak_rss_bytes": 34488320, + "hits": 4214, + "p50_ns": 3000, + "p99_ns": 3000, + "pair": 5 + } + ] + } + } + }, + "watcher": [ + { + "budget": 32, + "clients": 3, + "directories": 1200, + "source_files": 4800, + "phases": [ + { + "phase": "quiet", + "cpu_percent_per_client": [ + 0.6855512787589788, + 0.6569866421440215, + 0.5427280956841918 + ], + "wall_seconds": 35.008322125 + }, + { + "phase": "churn", + "cpu_percent_per_client": [ + 1.4569310020584922, + 1.3997964529581588, + 1.3712291784079924 + ], + "wall_seconds": 35.005089416 + } + ], + "unchanged_under_excluded_churn": true, + "alive": true, + "rss_kib": [ + 123856, + 84128, + 84992 + ], + "mutation_checks_passed": 61, + "mutation_checks_total": 61, + "failures": 0 + }, + { + "budget": 1024, + "clients": 1, + "directories": 1200, + "source_files": 4800, + "phases": [ + { + "phase": "quiet", + "cpu_percent_per_client": [ + 0.8854999240950605 + ], + "wall_seconds": 35.00847279199999 + }, + { + "phase": "churn", + "cpu_percent_per_client": [ + 1.5711053202346972 + ], + "wall_seconds": 35.007201167000005 + } + ], + "unchanged_under_excluded_churn": true, + "alive": true, + "rss_kib": [ + 105328 + ], + "mutation_checks_passed": 61, + "mutation_checks_total": 61, + "failures": 0 + } + ] +} diff --git a/docs/releases/0.2.5862.md b/docs/releases/0.2.5862.md new file mode 100644 index 00000000..c845b53a --- /dev/null +++ b/docs/releases/0.2.5862.md @@ -0,0 +1,78 @@ +# CodeDB 0.2.5862 + +Enables the Zig bbtree inline-key dictionary by default for warm mmap word +indexes. Exact lookup and prefix lower bounds use prefix-stripped inline keys, +vector counts, and tied-key comparisons; posting lists remain zero-copy in the +existing `word.index` mapping. The disk format is unchanged. + +## Memory and correctness limits + +All tree pages share one exact-size aligned slab, not one allocation per node. +The accelerator is capped at **8 MiB per mmap word index**. If the dictionary +would exceed that budget, or the allocation fails, CodeDB retains the complete +binary-search dictionary. This does not shrink retrieval breadth. The first +mutation frees the tree before unmapping and promoting to the existing mutable +heap index; no accelerator memory is retained after promotion. + +Ranking, query normalization, the hosted-default model, hosted exact fallback, +and ANN candidate breadth are unchanged. This release also includes the isolated +macOS watcher CPU fixes already on main through #758/#771 (derived from #750), +plus the verified-installer prerequisite in 0.2.5861. It deliberately does not +merge unrelated retrieval changes from the broader #750 branch. + +## Local measurements + +Native arm64 macOS, pinned Zig `0.17.0-dev.813+2153f8143`, ReleaseFast. The +comparison uses current source corpora from codedb, codegraff and zigrepper. +Sibling repositories and installed indexes are not modified. + +| Corpus | Dictionary words | Exact lookup paired improvement | Prefix search paired improvement | +| --- | ---: | ---: | ---: | +| codedb | 13,031 | 29.57% | 18.76% | +| codegraff | 42,100 | 44.70% | 13.24% | +| zigrepper | 36,994 | 30.24% | 11.47% | + +Seven alternating AB/BA rounds, pre-warmed mappings, production C allocator. +Percentages are medians of paired candidate/baseline ratios, not ratios of +independent medians. Timing is noisy; these are dictionary microbenchmarks, +not hosted or end-to-end context latency claims. + +Native CLI peak RSS was measured separately with six alternating pairs of +`bench-engine word function 10000`, isolated HOME directories, explicit empty +configuration, disabled telemetry/update/daemon/warmup, denied network access, +and disposable source copies. Both implementations returned the same nonzero +hit counts. Maximum resident bytes come from macOS `/usr/bin/time -l`. + +| Corpus | Median peak RSS increase | Relative increase | +| --- | ---: | ---: | +| codedb | 0.47 MiB | 4.84% | +| codegraff | 1.39 MiB | 3.10% | +| zigrepper | 1.21 MiB | 3.69% | + +All three stayed below the 5% relative and 10 MiB absolute acceptance guards. +This is bounded additional memory, not zero-cost memory: the single-word CLI +measurement is not whole-index residency or a long-lived daemon RSS guarantee. +The hard slab ceiling prevents growth with arbitrarily large dictionaries. + +## Verification + +- Full unit build: all 37 steps pass, including the default-on mmap path, + binary-search control, promote-on-write cleanup, allocation-failure/budget + cases, multilevel boundaries, and maximum disk-word prefix lengths. An + existing semantic retry timeout under concurrent compilation passed when + rerun without competing builds; unchanged successful suites were cached. +- ReleaseFast CLI build passes with the pinned release compiler. +- MCP E2E: **76/76** scenarios pass on the final frozen candidate. +- All exact lookups and lower bounds match across all three corpora; each + corpus also matches 1,000 full prefix results and all postings after promotion. +- Watcher stress: 4,800 files / 1,200 directories, 35-second quiet and churn + windows. Three clients with watch budget 32 measured 0.54–0.69% idle CPU each + and 1.37–1.46% under excluded churn. One client at budget 1,024 measured 0.89% + idle and 1.57% churn. All **122 mutation checks** passed; counters stayed + unchanged during excluded churn. 100% CPU is one core. + +Aggregate measurements and individual RSS samples are recorded in +[`../measurements/bbtree-0.2.5862.json`](../measurements/bbtree-0.2.5862.json). +Benchmark entry points: `experiments/bbtree/run.py` and +`experiments/bbtree/rss.py`. The source port retains upstream attribution and +MIT notice in `src/bbtree.LICENSE`. diff --git a/experiments/bbtree/LICENSE-upstream b/experiments/bbtree/LICENSE-upstream new file mode 100644 index 00000000..0300f934 --- /dev/null +++ b/experiments/bbtree/LICENSE-upstream @@ -0,0 +1,230 @@ +bbtree is licensed under either of the MIT License or the Apache License, +Version 2.0, at your option. Both texts follow. + +================================================================================ + +MIT License + +Copyright (c) 2026 Xiangpeng Hao + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/experiments/bbtree/README.md b/experiments/bbtree/README.md new file mode 100644 index 00000000..5f5c5a96 --- /dev/null +++ b/experiments/bbtree/README.md @@ -0,0 +1,109 @@ +# Warm mmap word dictionary: bbtree Zig experiment + +Historical experiment below. Release 0.2.5862 enables this read path by default +with a single slab and an 8 MiB cap; see +[release notes](../../docs/releases/0.2.5862.md) for RSS and verification results. + +## Result + +On this repository's tracked `src/*.zig` files (12,913 dictionary words), +the source-aligned bbtree read path improved warm exact lookup and prefix +search. It remains **opt-in**, not the production default: this is one local +corpus and a dictionary microbenchmark, not an end-to-end hybrid retrieval +measurement or a general performance claim. + +| Measurement | Binary-search median | bbtree median | Median paired latency change | +| --- | ---: | ---: | ---: | +| Exact `WordIndex.search` | 139.37 ns | 95.37 ns | -30.92% | +| Prefix `WordIndex.searchPrefix` (cap 100) | 430 ns | 370 ns | -14.42% | + +The paired percentage is the median of seven per-round candidate/baseline +ratios, not the ratio of the two independent medians. Added node payload: +435,712 bytes (~425.5 KiB), beyond the existing 51,652-byte offset directory. +Allocator overhead is not included. One-time tree construction: 0.188 ms. +Both mappings and the tree are warm before timing; cold paging/startup RSS +are not measured. + +## What was ported + +Pinned upstream: `XiangpengHao/bbtree` commit +`f0a5d361ecb4df176b3e07d4e33bb399dc3d22bf`. +The upstream license is retained in `LICENSE-upstream` (MIT option). + +`src/bbtree_dictionary.zig` specializes the **unsynchronized string read +algorithm**, not a generic B-tree substitute: + +- 1 KiB aligned leaves, 512-byte aligned internal pages; +- 8-byte leaf and 4-byte internal big-endian inline windows; +- full-capacity below/equal counts, expressed as Zig vector reductions; +- max-value padding, with live-length cuts; +- fence-derived prefix stripping and a root shared prefix capped at 64 bytes; +- separators held in right-child leaf heads, not copied full keys internally; +- early-exit binary search only within inline-window ties; +- saturated string-length tags and upstream's zero/one/two/many candidate cases; +- tied strings compared in 8-byte big-endian words, skipping proven-equal bytes. + +Source correspondence: upstream `tree/search.rs`, `tree/prefix.rs`, +`tree/shared_prefix.rs`, `tree/node/{inline_key_search,inline_keys,leaf,internal, +node_plan,slots,layout}.rs`, and `proof/search_bytes.rs`. + +This is a faithful **read-path specialization**, not a complete 1:1 library +translation. Deliberate integration adaptations: keys borrow validated mmap +records; values are dictionary ordinals; bulk construction balances the +already sorted dictionary directly instead of porting upstream's mutable +insertion/streaming builder. Zig generates the target's SIMD instructions +rather than copying Rust intrinsics. Concurrent latches, mutable tree updates, +file storage and the generic Rust API are not part of this experiment. + +## Integration and correctness + +Call `WordIndex.enableBBDictionary()` on a successfully mmap-loaded index to +opt in. Normal calls remain on the original binary-search path. Exact hits +still return the original zero-copy mmap postings. Prefix lookup changes only +the lower-bound search; forward scanning, strict-prefix exclusion, deduplication, +result caps and ordering are unchanged. First mutation frees the tree before +unmapping and promotes to the existing heap hash index. + +No disk-format, ranking, candidate-pool, embedding, hosted-fallback or query +normalization changes. No local embedding model is used. + +The benchmark checks all 38,739 exact queries and lower bounds against the +baseline, 1,000 full prefix results byte-for-byte, and every original posting +list after promotion. Queries comprise each real key, a missing extension, +and a half-length prefix, shuffled with a fixed seed. Timing alternates AB/BA +across seven rounds, five exact passes per round, with production `c_allocator`. +Source paths identify documents independently; the corpus is not collapsed +into a single synthetic document. Prefix timing includes allocations and hit +processing, not just tree descent. + +Standalone tests cover empty/singleton dictionaries, leaf and multilevel page +boundaries through 55,000 keys, long shared prefixes, prefix keys, inline ties, +and saturated length tags. Debug checks: 453 passed, 2 skipped across +`test-bbtree`, `test-index`, and `test-search`. The CLI ReleaseFast build also +passes. + +## Reproduce + +```sh +zig build test-bbtree test-index test-search --summary all +python3 experiments/bbtree/run.py +zig build -Doptimize=ReleaseFast --summary all +``` + +Measured on native Apple Silicon/macOS. The default installed Zig 0.17.0 is +incompatible with this checkout's build API. The pinned +`0.17.0-dev.813+2153f8143` archive returned HTTP 404; these runs used the +installed compatible `0.17.0-dev.1398+cb5635714`, without changing the global +compiler selection or migrating project code. To reproduce with that compiler: + +```sh +zigup run 0.17.0-dev.1398+cb5635714 build test-bbtree test-index test-search --summary all +ZIG_COMMAND='zigup run 0.17.0-dev.1398+cb5635714' python3 experiments/bbtree/run.py +zigup run 0.17.0-dev.1398+cb5635714 build -Doptimize=ReleaseFast --summary all +``` + +Original successful measurements are in `warm-results.txt`. Follow-up +three-repository measurements and existing-PR analysis are in +[cross-corpus-report.md](cross-corpus-report.md), with machine-readable results +in `cross-corpus-summary.json`. End-to-end retrieval and cold-load/RSS +measurements are still needed before enabling by default. diff --git a/experiments/bbtree/codedb-warm-results.txt b/experiments/bbtree/codedb-warm-results.txt new file mode 100644 index 00000000..6372d4e9 --- /dev/null +++ b/experiments/bbtree/codedb-warm-results.txt @@ -0,0 +1,16 @@ +files=63 source_bytes=3085391 words=12913 queries=38739 exact/lower-bound parity=all prefix_parity=1000 extra_bytes=435712 build_ms=0.149 +round=0 implementation=binary exact_ns=118.75 prefix_ns=403.00 +round=0 implementation=bbtree exact_ns=86.02 prefix_ns=344.00 +round=1 implementation=bbtree exact_ns=87.65 prefix_ns=335.00 +round=1 implementation=binary exact_ns=121.45 prefix_ns=397.00 +round=2 implementation=binary exact_ns=116.16 prefix_ns=413.00 +round=2 implementation=bbtree exact_ns=86.45 prefix_ns=337.00 +round=3 implementation=bbtree exact_ns=85.97 prefix_ns=352.00 +round=3 implementation=binary exact_ns=119.46 prefix_ns=405.00 +round=4 implementation=binary exact_ns=116.98 prefix_ns=401.00 +round=4 implementation=bbtree exact_ns=86.66 prefix_ns=340.00 +round=5 implementation=bbtree exact_ns=85.21 prefix_ns=332.00 +round=5 implementation=binary exact_ns=122.38 prefix_ns=397.00 +round=6 implementation=binary exact_ns=117.84 prefix_ns=402.00 +round=6 implementation=bbtree exact_ns=86.92 prefix_ns=345.00 +promotion_parity=all checksum=116683210 diff --git a/experiments/bbtree/codegraff-warm-results.txt b/experiments/bbtree/codegraff-warm-results.txt new file mode 100644 index 00000000..d7c81aae --- /dev/null +++ b/experiments/bbtree/codegraff-warm-results.txt @@ -0,0 +1,16 @@ +files=2119 source_bytes=16982003 words=42100 queries=126300 exact/lower-bound parity=all prefix_parity=1000 extra_bytes=1436672 build_ms=0.427 +round=0 implementation=binary exact_ns=166.07 prefix_ns=466.00 +round=0 implementation=bbtree exact_ns=104.82 prefix_ns=372.00 +round=1 implementation=bbtree exact_ns=103.55 prefix_ns=377.00 +round=1 implementation=binary exact_ns=163.36 prefix_ns=475.00 +round=2 implementation=binary exact_ns=162.57 prefix_ns=482.00 +round=2 implementation=bbtree exact_ns=132.17 prefix_ns=507.00 +round=3 implementation=bbtree exact_ns=137.38 prefix_ns=460.00 +round=3 implementation=binary exact_ns=176.34 prefix_ns=430.00 +round=4 implementation=binary exact_ns=175.89 prefix_ns=467.00 +round=4 implementation=bbtree exact_ns=110.87 prefix_ns=343.00 +round=5 implementation=bbtree exact_ns=107.69 prefix_ns=400.00 +round=5 implementation=binary exact_ns=181.14 prefix_ns=563.00 +round=6 implementation=binary exact_ns=195.90 prefix_ns=461.00 +round=6 implementation=bbtree exact_ns=107.90 prefix_ns=381.00 +promotion_parity=all checksum=1246317100 diff --git a/experiments/bbtree/cross-corpus-report.md b/experiments/bbtree/cross-corpus-report.md new file mode 100644 index 00000000..7ba3d29d --- /dev/null +++ b/experiments/bbtree/cross-corpus-report.md @@ -0,0 +1,113 @@ +# Cross-corpus warm mmap dictionary experiment + +Historical pre-release experiment: the opt-in/publication statements below +refer to that run. The subsequent 0.2.5862 release work makes the bounded +accelerator default; see [release notes](../../docs/releases/0.2.5862.md). + +## Outcome + +The opt-in Zig bbtree dictionary won the median paired comparison on all +three selected local corpora. No sibling repository was modified. No PR, +comment, commit or other publication was created. Production defaults remain +unchanged. + +| Corpus | Source files | Dictionary words | Exact median, binary → bbtree | Exact paired improvement | Prefix median, binary → bbtree | Prefix paired improvement | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| codedb | 63 | 12,913 | 118.75 → 86.45 ns | 27.56% | 402 → 340 ns | 15.21% | +| codegraff | 2,119 | 42,100 | 175.89 → 107.90 ns | 36.88% | 467 → 381 ns | 20.17% | +| zigrepper | 1,548 | 36,994 | 155.44 → 101.93 ns | 33.98% | 470 → 391 ns | 14.66% | + +Improvement is `1 - median(candidate_round / baseline_round)` over seven +paired rounds; it is not computed from the independent latency medians. +Individual rounds are noisy: codegraff prefix search was slower in two rounds, +although its paired median improved. These numbers are microbenchmark latency, +not CLI/MCP or hosted hybrid end-to-end latency. + +| Corpus | Added node payload | Tree construction | +| --- | ---: | ---: | +| codedb | 435,712 bytes | 0.149 ms | +| codegraff | 1,436,672 bytes | 0.427 ms | +| zigrepper | 1,262,592 bytes | 0.361 ms | + +Node payload excludes allocator overhead and the original offset directory. +Both binary and candidate paths retain zero-copy mmap postings. Timed queries +run only after parity walks have warmed both mappings and tree nodes. + +## Corpus selection and correctness + +The codedb control keeps the original tracked `src/*.zig` selection. The other +corpora use tracked source files across supported/common source extensions, +excluding hidden/generated/vendor/build directories, symlinked paths, missing +files, and files over 64 MiB. Ten zigrepper source candidates were skipped by +these safety checks. Sources come from current working trees, not immutable +commit snapshots; codedb and codegraff were dirty, zigrepper was clean. Existing +changes were retained. No untracked sibling source files were included. + +This builds fresh temporary `WordIndex` fixtures from the selected sources; +it does not modify or benchmark a running installed daemon or reuse its private +index. All fixture data stays in codedb's `.zig-cache` and is removed after each +successful run. Result files contain aggregate metrics, not source text or +absolute corpus paths. No embeddings, network retrieval, or local models run. + +- codedb: **38,739** exact queries and lower bounds checked. +- codegraff: **126,300** exact queries and lower bounds checked. +- zigrepper: **110,982** exact queries and lower bounds checked. +- Every corpus: **1,000 full prefix results**, byte-for-byte identical. +- Every corpus: **all original posting lists** identical after mutation promotes + the candidate back to the existing mutable heap index. + +The actual `WordIndex.search` and `searchPrefix` methods are timed with the +production C allocator. Prefix timing includes result allocation and hit +processing. Seven AB/BA rounds run per corpus, with five full exact-query passes +per round and the same deterministic shuffled query mix as the first experiment. + +## Existing PRs inspected + +- [codedb #525](https://github.com/justrach/codedb/pull/525), **merged**: + introduced the zero-copy mmap word index and warm CLI daemon. This is the + direct architectural predecessor: the experiment optimizes dictionary lookup + inside that path, preserving postings and promote-on-write behavior. +- [codedb #551](https://github.com/justrach/codedb/pull/551), **merged**: + loads the word index for CLI search. The new tree could accelerate its word + lookup once enabled; it does not replace that recall/correctness fix. +- [codedb #607](https://github.com/justrach/codedb/pull/607), **merged**: + fixes document-ID reuse and persistence attribution. Preserve it; bbtree does + not change document IDs or substitute for its mutation logic. +- [codedb #750](https://github.com/justrach/codedb/pull/750), **open**: + semantic fallback and watcher work; touches related tests but not + `src/index.zig`. It is not a competing dictionary implementation to migrate. +- The current open codedb PR file lists contain no `src/index.zig` change. +- [zigrepper #155](https://github.com/justrach/zigrepper/pull/155), **open**: + codedbpro lookup routing and compact MCP catalog. It changes dispatch/tool + overhead, not the native mmap dictionary. The benchmark does not establish + that its own lookup implementation should be replaced by bbtree. +- Its linked [codegraff #603](https://github.com/justrach/codegraff/pull/603) + is **closed**; it concerns one-shot routing and the compact catalog, not + dictionary storage. It is not treated as merged. + +No inspected PR proposes migrating an ordered map to bbtree. The evidence +supports further adoption evaluation in native codedb's mmap word dictionary, +not asking these unrelated PRs to switch algorithms. + +## Reproduction and next decision + +Run from this checkout, with explicit read-only corpus roots: + +```sh +ZIG_COMMAND='zigup run 0.17.0-dev.1398+cb5635714' \ + python3 experiments/bbtree/run.py \ + --corpus codedb "$PWD" \ + --corpus codegraff /absolute/path/to/codegraff \ + --corpus zigrepper /absolute/path/to/zigrepper +``` + +The compatible compiler and native Apple Silicon/macOS environment match the +first experiment. Raw paired rounds are in `*-warm-results.txt`; the runner's +computed medians and corpus metadata are in `cross-corpus-summary.json`. +The old `warm-results.txt` retains the original single-corpus run. + +This broadens the dictionary evidence from one to three repositories. Before +making it default, measure end-to-end native retrieval and resident memory/cold +loading: the absolute lookup savings are tens of nanoseconds and the additional +tree uses roughly 0.4–1.4 MiB on these corpora. Ranking, candidate breadth, hosted +fallback, and retrieval accuracy must remain unchanged. diff --git a/experiments/bbtree/cross-corpus-summary.json b/experiments/bbtree/cross-corpus-summary.json new file mode 100644 index 00000000..e632f298 --- /dev/null +++ b/experiments/bbtree/cross-corpus-summary.json @@ -0,0 +1,71 @@ +{ + "codedb": { + "exact": { + "binary_median_ns": 118.75, + "bbtree_median_ns": 86.45, + "paired_latency_change_pct": -27.5621052631579 + }, + "prefix": { + "binary_median_ns": 402.0, + "bbtree_median_ns": 340.0, + "paired_latency_change_pct": -15.211970074812964 + }, + "corpus": { + "files": 63, + "source_bytes": 3085391, + "words": 12913, + "queries": 38739, + "prefix_parity": 1000, + "extra_bytes": 435712, + "build_ms": 0.149 + }, + "working_tree_dirty": true, + "skipped_source_files": 0 + }, + "codegraff": { + "exact": { + "binary_median_ns": 175.89, + "bbtree_median_ns": 107.9, + "paired_latency_change_pct": -36.88203769494791 + }, + "prefix": { + "binary_median_ns": 467.0, + "bbtree_median_ns": 381.0, + "paired_latency_change_pct": -20.171673819742487 + }, + "corpus": { + "files": 2119, + "source_bytes": 16982003, + "words": 42100, + "queries": 126300, + "prefix_parity": 1000, + "extra_bytes": 1436672, + "build_ms": 0.427 + }, + "working_tree_dirty": false, + "skipped_source_files": 0 + }, + "zigrepper": { + "exact": { + "binary_median_ns": 155.44, + "bbtree_median_ns": 101.93, + "paired_latency_change_pct": -33.979378752284006 + }, + "prefix": { + "binary_median_ns": 470.0, + "bbtree_median_ns": 391.0, + "paired_latency_change_pct": -14.660831509846828 + }, + "corpus": { + "files": 1548, + "source_bytes": 12160841, + "words": 36994, + "queries": 110982, + "prefix_parity": 1000, + "extra_bytes": 1262592, + "build_ms": 0.361 + }, + "working_tree_dirty": true, + "skipped_source_files": 10 + } +} diff --git a/experiments/bbtree/rss.py b/experiments/bbtree/rss.py new file mode 100644 index 00000000..3ec3184e --- /dev/null +++ b/experiments/bbtree/rss.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Compare frozen native CLI RSS using disposable copies, never original indexes.""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import statistics +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('bb_runner', Path(__file__).with_name('run.py')) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline', required=True, type=Path) + parser.add_argument('--candidate', required=True, type=Path) + parser.add_argument('--corpus', nargs=2, action='append', required=True) + parser.add_argument('--out', required=True, type=Path) + args = parser.parse_args() + binaries = {'binary': args.baseline.resolve(), 'bbtree': args.candidate.resolve()} + results = {} + cache = ROOT / '.zig-cache/release-verify' + cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='rss-', dir=cache) as tmp: + work = Path(tmp) + config = work / 'empty.codedbrc' + config.write_text('') + for name, original in args.corpus: + source = Path(original).resolve() + paths, skipped = runner.corpus_paths(source, default=name == 'codedb' and source == ROOT) + corpus = work / name + corpus.mkdir() + for path in paths: + destination = corpus / path.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, destination) + subprocess.run(['git', 'init', '-q', str(corpus)], check=True) + rows = [] + def measure(side): + home = work / ('home-' + name + '-' + side) + home.mkdir(exist_ok=True) + env = {'PATH': os.environ['PATH'], 'HOME': str(home), 'TMPDIR': str(work), + 'CODEDB_NO_TELEMETRY': '1', 'CODEDB_NO_AUTO_UPDATE': '1', + 'CODEDB_NO_CLI_DAEMON': '1', 'CODEDB_NO_WARMUP': '1', + 'CODEDB_ALLOW_TEMP': '1'} + # Deny network access, even if a future command accidentally + # starts a semantic provider. No credentials enter this process. + command = ['/usr/bin/sandbox-exec', '-p', '(version 1)(allow default)(deny network*)', + '/usr/bin/time', '-l', str(binaries[side]), str(corpus), + 'bench-engine', 'word', 'function', '10000', + '--config-file=' + str(config)] + result = subprocess.run(command, env=env, cwd=corpus, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180) + if result.returncode: + raise RuntimeError('Native RSS command failed: ' + result.stderr[-2000:]) + metrics = [json.loads(line) for line in result.stdout.splitlines() if line.startswith('{')] + timing = metrics[-1] + match = re.search(r'(\d+)\s+maximum resident set size', result.stderr) + if not match: + raise RuntimeError('Missing macOS time peak RSS') + return {'implementation': side, 'peak_rss_bytes': int(match.group(1)), + 'hits': timing['hits'], 'p50_ns': timing['p50_ns'], 'p99_ns': timing['p99_ns']} + # Prepare each HOME/index separately, then only compare warm loads. + for side in binaries: + measure(side) + for pair in range(6): + order = ['binary', 'bbtree'] if pair % 2 == 0 else ['bbtree', 'binary'] + pair_rows = [] + for side in order: + row = measure(side) + row['pair'] = pair + rows.append(row) + pair_rows.append(row) + if pair_rows[0]['hits'] != pair_rows[1]['hits'] or pair_rows[0]['hits'] == 0: + raise RuntimeError('Native CLI word-result parity failed or query has no hits') + medians = {side: statistics.median(r['peak_rss_bytes'] for r in rows + if r['implementation'] == side) for side in binaries} + delta = medians['bbtree'] - medians['binary'] + results[name] = {'source_files': len(paths), 'skipped_source_files': skipped, + 'binary_median_peak_rss_bytes': medians['binary'], + 'bbtree_median_peak_rss_bytes': medians['bbtree'], + 'median_peak_rss_delta_bytes': delta, + 'median_peak_rss_delta_pct': 100 * delta / medians['binary'], + 'samples': rows} + print(json.dumps({name: {k: v for k, v in results[name].items() if k != 'samples'}}), flush=True) + args.out.write_text(json.dumps(results, indent=2) + '\n') + # Fixed absolute ceiling and relative guard prevent an unnoticed RSS blowup. + if any(r['median_peak_rss_delta_bytes'] > 10 * 1024 * 1024 or + r['median_peak_rss_delta_pct'] > 5 for r in results.values()): + raise SystemExit('RSS regression exceeds 10 MiB absolute or 5% relative guard') + + +if __name__ == '__main__': + main() diff --git a/experiments/bbtree/run.py b/experiments/bbtree/run.py new file mode 100644 index 00000000..e8848fba --- /dev/null +++ b/experiments/bbtree/run.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Paired warm mmap benchmarks; external corpora are read-only and explicit.""" +import argparse +import json +import os +from pathlib import Path +import re +import shlex +import statistics +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE_SUFFIXES = {'.zig', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', + '.rs', '.py', '.go', '.c', '.h', '.cc', '.cpp', '.hpp', + '.swift', '.rb', '.php', '.java', '.kt', '.cs', '.dart'} +SKIP_DIRS = {'node_modules', 'vendor', 'third_party', 'zig-out', 'zig-cache', + 'target', 'dist', 'build', '__pycache__', 'generated'} + + +def corpus_paths(directory, default=False): + command = ['git', '-C', str(directory), 'ls-files', '-z'] + if default: + command += ['--', 'src'] + paths = [] + skipped = 0 + for raw in subprocess.check_output(command).split(b'\0'): + if not raw: + continue + relative = Path(os.fsdecode(raw)) + if relative.suffix not in ({'.zig'} if default else SOURCE_SUFFIXES): + continue + if any(part in SKIP_DIRS or part.startswith('.') for part in relative.parts[:-1]): + continue + path = directory / relative + # Never follow symlinks out of the selected corpus, including parents. + resolved = path.resolve() + try: + resolved.relative_to(directory) + except ValueError: + skipped += 1 + continue + if resolved != path or not path.is_file(): + skipped += 1 + continue + if path.stat().st_size > 64 * 1024 * 1024: + skipped += 1 + continue + paths.append(path) + if not paths: + raise ValueError('No eligible tracked source files') + return paths, skipped + + +def summarize(text): + rows = re.findall(r'round=(\d+) implementation=(binary|bbtree) exact_ns=([\d.]+) prefix_ns=([\d.]+)', text) + if len(rows) != 14: + raise ValueError('Expected seven complete paired rounds') + summary = {} + for column, label in [(2, 'exact'), (3, 'prefix')]: + medians = {name: statistics.median(float(row[column]) for row in rows + if row[1] == name) + for name in ['binary', 'bbtree']} + ratios = [] + for round_id in sorted(set(row[0] for row in rows)): + pair = {row[1]: float(row[column]) for row in rows if row[0] == round_id} + ratios.append(pair['bbtree'] / pair['binary']) + summary[label] = {'binary_median_ns': medians['binary'], + 'bbtree_median_ns': medians['bbtree'], + 'paired_latency_change_pct': (statistics.median(ratios) - 1) * 100} + fields = dict(re.findall(r'(files|source_bytes|words|queries|prefix_parity|extra_bytes|build_ms)=([\d.]+)', text)) + summary['corpus'] = {key: float(value) if key == 'build_ms' else int(value) + for key, value in fields.items()} + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--corpus', nargs=2, action='append', metavar=('NAME', 'ROOT')) + parser.add_argument('--output-dir', type=Path, default=ROOT / 'experiments/bbtree') + args = parser.parse_args() + os.chdir(ROOT) + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=True) + corpora = args.corpus or [('codedb', str(ROOT))] + zig = shlex.split(os.environ.get('ZIG_COMMAND', 'zig')) + results = {} + for name, directory in corpora: + if not re.fullmatch(r'[A-Za-z0-9_-]+', name) or name in results: + parser.error('Corpus labels must be unique simple names') + directory = Path(directory).resolve() + # Record dirty status only as a boolean, never source paths or git output. + dirty = bool(subprocess.check_output(['git', '-C', str(directory), 'status', '--porcelain'])) + paths, skipped = corpus_paths(directory, default=name == 'codedb' and directory == ROOT) + cache = ROOT / '.zig-cache' + cache.mkdir(exist_ok=True) + with tempfile.NamedTemporaryFile(dir=cache) as manifest: + manifest.write(b'\0'.join(os.fsencode(path) for path in paths) + b'\0') + manifest.flush() + command = zig + ['build', 'bench-bbtree', '-Doptimize=ReleaseFast', '--', '--manifest', manifest.name] + print('Benchmarking {}: {} source files, skipped={}'.format(name, len(paths), skipped), flush=True) + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + sys.stdout.write(result.stdout) + if result.returncode: + sys.exit(result.returncode) + summary = summarize(result.stdout) + summary.update({'working_tree_dirty': dirty, 'skipped_source_files': skipped}) + results[name] = summary + # Successful aggregate measurements only: no absolute paths or source text. + filename = 'warm-results.txt' if not args.corpus else name + '-warm-results.txt' + (output / filename).write_text(result.stdout) + for operation in ['exact', 'prefix']: + row = summary[operation] + print('{} {}: {:.2f} -> {:.2f} ns; paired latency change {:+.2f}%'.format( + name, operation, row['binary_median_ns'], row['bbtree_median_ns'], row['paired_latency_change_pct'])) + if args.corpus: + (output / 'cross-corpus-summary.json').write_text(json.dumps(results, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/experiments/bbtree/warm-results.txt b/experiments/bbtree/warm-results.txt new file mode 100644 index 00000000..2ae653b6 --- /dev/null +++ b/experiments/bbtree/warm-results.txt @@ -0,0 +1,16 @@ +words=12913 queries=38739 exact/lower-bound parity=all prefix_parity=1000 extra_bytes=435712 build_ms=0.188 +round=0 implementation=binary exact_ns=151.67 prefix_ns=542.00 +round=0 implementation=bbtree exact_ns=94.21 prefix_ns=359.00 +round=1 implementation=bbtree exact_ns=95.26 prefix_ns=368.00 +round=1 implementation=binary exact_ns=137.90 prefix_ns=430.00 +round=2 implementation=binary exact_ns=139.37 prefix_ns=445.00 +round=2 implementation=bbtree exact_ns=96.76 prefix_ns=389.00 +round=3 implementation=bbtree exact_ns=96.13 prefix_ns=379.00 +round=3 implementation=binary exact_ns=143.50 prefix_ns=484.00 +round=4 implementation=binary exact_ns=132.16 prefix_ns=410.00 +round=4 implementation=bbtree exact_ns=95.37 prefix_ns=370.00 +round=5 implementation=bbtree exact_ns=87.07 prefix_ns=352.00 +round=5 implementation=binary exact_ns=145.00 prefix_ns=426.00 +round=6 implementation=binary exact_ns=135.34 prefix_ns=424.00 +round=6 implementation=bbtree exact_ns=103.11 prefix_ns=380.00 +promotion_parity=all checksum=116683210 diff --git a/experiments/bbtree/zigrepper-warm-results.txt b/experiments/bbtree/zigrepper-warm-results.txt new file mode 100644 index 00000000..c298fdc2 --- /dev/null +++ b/experiments/bbtree/zigrepper-warm-results.txt @@ -0,0 +1,16 @@ +files=1548 source_bytes=12160841 words=36994 queries=110982 exact/lower-bound parity=all prefix_parity=1000 extra_bytes=1262592 build_ms=0.361 +round=0 implementation=binary exact_ns=183.53 prefix_ns=505.00 +round=0 implementation=bbtree exact_ns=101.93 prefix_ns=391.00 +round=1 implementation=bbtree exact_ns=101.17 prefix_ns=390.00 +round=1 implementation=binary exact_ns=153.24 prefix_ns=457.00 +round=2 implementation=binary exact_ns=151.09 prefix_ns=459.00 +round=2 implementation=bbtree exact_ns=109.06 prefix_ns=418.00 +round=3 implementation=bbtree exact_ns=125.97 prefix_ns=418.00 +round=3 implementation=binary exact_ns=155.44 prefix_ns=470.00 +round=4 implementation=binary exact_ns=155.03 prefix_ns=474.00 +round=4 implementation=bbtree exact_ns=95.87 prefix_ns=380.00 +round=5 implementation=bbtree exact_ns=96.98 prefix_ns=387.00 +round=5 implementation=binary exact_ns=164.89 prefix_ns=482.00 +round=6 implementation=binary exact_ns=161.17 prefix_ns=465.00 +round=6 implementation=bbtree exact_ns=108.84 prefix_ns=439.00 +promotion_parity=all checksum=757345946 diff --git a/npm/package.json b/npm/package.json index 028b7a45..d2e230e2 100644 --- a/npm/package.json +++ b/npm/package.json @@ -1,6 +1,6 @@ { "name": "codedeebee", - "version": "0.2.5861", + "version": "0.2.5862", "description": "Zig code intelligence MCP server — npx launcher for the codedb native binary", "license": "MIT", "author": "justrach", diff --git a/src/bbtree.LICENSE b/src/bbtree.LICENSE new file mode 100644 index 00000000..f52012d4 --- /dev/null +++ b/src/bbtree.LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Xiangpeng Hao + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/bbtree_dictionary.zig b/src/bbtree_dictionary.zig new file mode 100644 index 00000000..5204e4bc --- /dev/null +++ b/src/bbtree_dictionary.zig @@ -0,0 +1,293 @@ +//! Read-only specialization of XiangpengHao/bbtree's unsync string search. +//! Copyright (c) 2026 Xiangpeng Hao; MIT notice: bbtree.LICENSE. +//! Upstream f0a5d361ecb4df176b3e07d4e33bb399dc3d22bf (MIT): +//! tree/{search,prefix}.rs, node/{inline_key_search,leaf,internal,node_plan}.rs. +//! Same 1KiB leaves / 512B internals, 8/4-byte inline windows, fence prefixes, +//! full-capacity counts, head separators, early-exit tie search and length tags. +//! Keys borrow validated mmap records; values are sorted dictionary ordinals. +//! Only bulk construction is specialized: no mutable/concurrent map is ported. +const std = @import("std"); + +const Entry = extern struct { key: [*]const u8, len: usize, rank: u32, padding: u32 = 0 }; +const Header = extern struct { len: u16, prefix: u16, reserved: u32 = 0 }; +const Leaf = extern struct { + header: Header align(1024), + windows: [30]u64, + tags: [30]u8, + padding: [2]u8 = .{ 0, 0 }, + entries: [31]Entry, // slot zero is the separator's head +}; +const Internal = extern struct { + header: Header align(512), + windows: [41]u32, + padding: u32 = 0, + children: [42]usize, +}; +comptime { + std.debug.assert(@sizeOf(Leaf) == 1024); + std.debug.assert(@sizeOf(Internal) == 512); +} + +pub const Dictionary = struct { + allocator: std.mem.Allocator, + root: usize = 0, // low bit denotes an internal node + shared: []const u8 = &.{}, + count: usize, + bytes: usize = 0, + storage: []align(1024) u8 = &.{}, + leaf_cursor: usize = 0, + internal_cursor: usize = 0, + + /// A fixed ceiling on accelerator memory, independent of corpus size. + /// Larger dictionaries retain the complete binary-search index instead. + pub const max_bytes: usize = 8 * 1024 * 1024; + + pub fn init(allocator: std.mem.Allocator, data: []const u8, offsets: []const u32) !Dictionary { + return initWithBudget(allocator, data, offsets, max_bytes); + } + + pub fn initWithBudget(allocator: std.mem.Allocator, data: []const u8, offsets: []const u32, budget: usize) !Dictionary { + var self = Dictionary{ .allocator = allocator, .count = offsets.len }; + if (offsets.len == 0) return self; + // Each record needs at least one Entry, even before inline windows. + // Reject oversized input before traversing its shape or reading keys. + if (offsets.len > budget / @sizeOf(Entry)) return error.DictionaryBudgetExceeded; + const first = word(data, offsets[0]); + self.shared = first[0..@min(64, common(first, word(data, offsets[offsets.len - 1])))]; + var height: usize = 0; + var capacity: usize = 30; + while (capacity < offsets.len) : (height += 1) capacity = capacity * 42 + 41; + const layout = nodeCounts(offsets.len, height); + self.bytes = layout.leaves * @sizeOf(Leaf) + layout.internals * @sizeOf(Internal); + if (self.bytes > budget) return error.DictionaryBudgetExceeded; + // One exact-size allocation: no per-node malloc/alignment overhead and + // no arena high-water retention after promotion. Leaves come first so + // every leaf and internal page has its upstream alignment without gaps. + self.storage = try allocator.alignedAlloc(u8, .fromByteUnits(1024), self.bytes); + self.internal_cursor = layout.leaves * @sizeOf(Leaf); + self.root = self.build(data, offsets, 0, offsets.len, height, null, null, null); + return self; + } + + pub fn deinit(self: *Dictionary) void { + self.allocator.free(self.storage); + } + + fn nodeCounts(count: usize, height: usize) struct { leaves: usize, internals: usize } { + if (height == 0) return .{ .leaves = 1, .internals = 0 }; + var capacity: usize = 30; + for (1..height) |_| capacity = capacity * 42 + 41; + const children = @min(42, (count + capacity + 1) / (capacity + 1)); + const remaining = count - (children - 1); + var total: @TypeOf(nodeCounts(0, 0)) = .{ .leaves = 0, .internals = 1 }; + for (0..children) |i| { + const child = nodeCounts(remaining / children + @intFromBool(i < remaining % children), height - 1); + total.leaves += child.leaves; + total.internals += child.internals; + } + return total; + } + + fn build(self: *Dictionary, data: []const u8, offsets: []const u32, start: usize, end: usize, height: usize, head: ?usize, low: ?usize, high: ?usize) usize { + const prefix = if (low != null and high != null) + common(word(data, offsets[low.?]), word(data, offsets[high.?])) + else + self.shared.len; + if (height == 0) { + const n: *Leaf = @ptrCast(@alignCast(self.storage.ptr + self.leaf_cursor)); + self.leaf_cursor += @sizeOf(Leaf); + n.* = .{ .header = .{ .len = @intCast(end - start), .prefix = @intCast(prefix) }, .windows = @splat(std.math.maxInt(u64)), .tags = @splat(255), .entries = undefined }; + if (head) |h| n.entries[0] = entry(data, offsets, h); + for (start..end, 0..) |rank, i| { + const e = entry(data, offsets, rank); + n.entries[i + 1] = e; + n.windows[i] = window(e.key[0..e.len], prefix); + n.tags[i] = tag(e.len); + } + return @intFromPtr(n); + } + var child_capacity: usize = 30; + for (1..height) |_| child_capacity = child_capacity * 42 + 41; + const count = end - start; + const children = @min(42, (count + child_capacity + 1) / (child_capacity + 1)); + const n: *Internal = @ptrCast(@alignCast(self.storage.ptr + self.internal_cursor)); + self.internal_cursor += @sizeOf(Internal); + n.* = .{ .header = .{ .len = @intCast(children - 1), .prefix = @intCast(prefix) }, .windows = @splat(std.math.maxInt(u32)), .children = undefined }; + const remaining = count - (children - 1); + var pos = start; + var prev = low; + for (0..children) |i| { + const size = remaining / children + @intFromBool(i < remaining % children); + const next: ?usize = if (i + 1 < children) pos + size else high; + n.children[i] = self.build(data, offsets, pos, pos + size, height - 1, if (i == 0) head else prev, prev, next); + pos += size; + if (i + 1 < children) { + n.windows[i] = @truncate(window(word(data, offsets[pos]), prefix) >> 32); + prev = pos; + pos += 1; + } + } + return @intFromPtr(n) | 1; + } + + /// Ordered lookup, also used as the prefix range's lower bound. + pub fn lowerBound(self: *const Dictionary, key: []const u8) usize { + if (self.root == 0) return 0; + const compared = std.mem.order(u8, key[0..@min(key.len, self.shared.len)], self.shared); + if (compared == .lt) return 0; + if (compared == .gt) return self.count; + var node = self.root; + while (node & 1 != 0) { + const n = internal(node); + const target: u32 = @truncate(window(key, n.header.prefix) >> 32); + const range = equalRange(u32, 41, n.windows, n.header.len, target); + var lo = range[0]; + var hi = range[1]; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + const e = headEntry(n.children[mid + 1]); + switch (compare(key, e, @as(usize, n.header.prefix) + 4)) { + .eq => return e.rank, + .lt => hi = mid, + .gt => lo = mid + 1, + } + } + node = n.children[lo]; + } + const n = leaf(node); + const range = equalRange(u64, 30, n.windows, n.header.len, window(key, n.header.prefix)); + var lo = range[0]; + var hi = range[1]; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + switch (compare(key, n.entries[mid + 1], @as(usize, n.header.prefix) + 8)) { + .eq => return n.entries[mid + 1].rank, + .lt => hi = mid, + .gt => lo = mid + 1, + } + } + if (lo < n.header.len) return n.entries[lo + 1].rank; + if (n.header.len > 0) return n.entries[n.header.len].rank + 1; + return n.entries[0].rank + 1; + } + + pub fn find(self: *const Dictionary, key: []const u8) ?usize { + if (self.root == 0 or !std.mem.startsWith(u8, key, self.shared)) return null; + var node = self.root; + while (node & 1 != 0) { + const n = internal(node); + const range = equalRange(u32, 41, n.windows, n.header.len, @truncate(window(key, n.header.prefix) >> 32)); + var lo = range[0]; + var hi = range[1]; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + const e = headEntry(n.children[mid + 1]); + switch (compare(key, e, @as(usize, n.header.prefix) + 4)) { + .eq => return e.rank, + .lt => hi = mid, + .gt => lo = mid + 1, + } + } + node = n.children[lo]; + } + const n = leaf(node); + const range = equalRange(u64, 30, n.windows, n.header.len, window(key, n.header.prefix)); + if (range[0] == range[1]) return null; + if (range[1] - range[0] == 1) { + const e = n.entries[range[0] + 1]; + return if (compare(key, e, @as(usize, n.header.prefix) + 8) == .eq) e.rank else null; + } + var matches: [2]usize = undefined; + var count: usize = 0; + for (range[0]..range[1]) |i| { + if (n.tags[i] == tag(key.len)) { + if (count < 2) matches[count] = i; + count += 1; + } + } + if (count <= 2) { + var i = count; + while (i > 0) { + i -= 1; + const e = n.entries[matches[i] + 1]; + const order = compare(key, e, @as(usize, n.header.prefix) + 8); + if (order == .eq) return e.rank; + if (order == .gt) return null; + } + return null; + } + var lo = range[0]; + var hi = range[1]; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + const e = n.entries[mid + 1]; + switch (compare(key, e, @as(usize, n.header.prefix) + 8)) { + .eq => return e.rank, + .lt => hi = mid, + .gt => lo = mid + 1, + } + } + return null; + } +}; + +fn leaf(node: usize) *Leaf { + return @ptrFromInt(node); +} +fn internal(node: usize) *Internal { + return @ptrFromInt(node & ~@as(usize, 1)); +} +fn headEntry(node_: usize) Entry { + var node = node_; + while (node & 1 != 0) node = internal(node).children[0]; + return leaf(node).entries[0]; +} +pub fn word(data: []const u8, offset: u32) []const u8 { + const len = std.mem.readInt(u16, data[offset..][0..2], .little); + return data[offset + 2 ..][0..len]; +} +fn entry(data: []const u8, offsets: []const u32, rank: usize) Entry { + const key = word(data, offsets[rank]); + return .{ .key = key.ptr, .len = key.len, .rank = @intCast(rank) }; +} +fn common(a: []const u8, b: []const u8) usize { + var i: usize = 0; + while (i < @min(a.len, b.len) and a[i] == b[i]) : (i += 1) {} + return i; +} +fn tag(len: usize) u8 { + return @intCast(@min(255, len)); +} +fn window(key: []const u8, prefix: usize) u64 { + var bytes: [8]u8 = @splat(0); + if (prefix < key.len) @memcpy(bytes[0..@min(8, key.len - prefix)], key[prefix..][0..@min(8, key.len - prefix)]); + return std.mem.readInt(u64, &bytes, .big); +} +fn compare(key: []const u8, e: Entry, skip: usize) std.math.Order { + // Upstream SearchBytes::cmp_from: compare 8-byte big-endian words; + // the final word may overlap bytes already proven equal. No memcmp. + const stored = e.key[0..e.len]; + const both = @min(key.len, e.len); + if (skip < both and both < 8) { + const order = std.math.order(window(key, 0), window(stored, 0)); + return if (order == .eq) std.math.order(key.len, e.len) else order; + } + var compared = skip; + while (compared < both) { + const end = @min(compared + 8, both); + const query_word = std.mem.readInt(u64, key[end - 8 ..][0..8], .big); + const stored_word = std.mem.readInt(u64, stored[end - 8 ..][0..8], .big); + const order = std.math.order(query_word, stored_word); + if (order != .eq) return order; + compared = end; + } + return std.math.order(key.len, e.len); +} +fn equalRange(comptime T: type, comptime capacity: usize, slots: [capacity]T, len: usize, target: T) [2]usize { + const values: @Vector(capacity, T) = slots; + const query: @Vector(capacity, T) = @splat(target); + const below = @reduce(.Add, @as(@Vector(capacity, u16), @intFromBool(values < query))); + const equal = @reduce(.Add, @as(@Vector(capacity, u16), @intFromBool(values == query))); + const start = @min(len, below); + return .{ start, @min(len, start + equal) }; +} diff --git a/src/bench_bbtree.zig b/src/bench_bbtree.zig new file mode 100644 index 00000000..7daf9f64 --- /dev/null +++ b/src/bench_bbtree.zig @@ -0,0 +1,117 @@ +//! Paired warm mmap dictionary experiment. No embeddings or network calls. +const std = @import("std"); +const cio = @import("cio.zig"); +const idx = @import("index.zig"); +const bb = @import("bbtree_dictionary.zig"); + +pub fn main(init: std.process.Init.Minimal) !void { + cio.setProcessArgs(cio.bootstrapArgs(init.args)); + // Match the CLI's production allocator, including prefix-result allocations. + const a = std.heap.c_allocator; + var threaded: std.Io.Threaded = .init(a, .{}); + defer threaded.deinit(); + const io = threaded.io(); + const args = try cio.argsAlloc(a); + defer cio.argsFree(a, args); + if (args.len < 2) return error.ExpectedSourceFiles; + var manifest: ?[]u8 = null; + defer if (manifest) |bytes| a.free(bytes); + var paths: std.ArrayList([]const u8) = .empty; + defer paths.deinit(a); + if (args.len == 3 and std.mem.eql(u8, args[1], "--manifest")) { + manifest = try std.Io.Dir.cwd().readFileAlloc(io, args[2], a, .limited(64 * 1024 * 1024)); + var entries = std.mem.splitScalar(u8, manifest.?, 0); + while (entries.next()) |path| if (path.len > 0) { + try paths.append(a, path); + }; + } else try paths.appendSlice(a, args[1..]); + if (paths.items.len == 0) return error.ExpectedSourceFiles; + var source_bytes: usize = 0; + var heap = idx.WordIndex.init(a); + defer heap.deinit(); + for (paths.items) |path| { + const content = try std.Io.Dir.cwd().readFileAlloc(io, path, a, .limited(64 * 1024 * 1024)); + defer a.free(content); + try heap.indexFile(path, content); + source_bytes += content.len; + } + const dir = try std.fmt.allocPrint(a, ".zig-cache/bbtree-bench-{d}", .{cio.nanoTimestamp()}); + defer a.free(dir); + try std.Io.Dir.cwd().createDirPath(io, dir); + defer std.Io.Dir.cwd().deleteTree(io, dir) catch {}; + try heap.writeToDisk(io, dir, null); + var baseline = idx.WordIndex.mmapFromDiskWithDictionary(io, dir, a, false) orelse return error.MmapLoadFailed; + defer baseline.deinit(); + var candidate = idx.WordIndex.mmapFromDiskWithDictionary(io, dir, a, false) orelse return error.MmapLoadFailed; + defer candidate.deinit(); + const build_start = cio.nanoTimestamp(); + try candidate.enableBBDictionary(); + const build_ns = cio.nanoTimestamp() - build_start; + const data = baseline.mmap_data.?; + var queries: std.ArrayList([]const u8) = .empty; + defer { + for (queries.items) |q| a.free(q); + queries.deinit(a); + } + // Every real dictionary key, an absent extension, and a prefix (hit or miss). + for (baseline.word_dir) |off| { + const key = bb.word(data, off); + try queries.append(a, try a.dupe(u8, key)); + try queries.append(a, try std.fmt.allocPrint(a, "{s}~absent", .{key})); + try queries.append(a, try a.dupe(u8, key[0 .. key.len / 2])); + } + // Fixed shuffle prevents sorted query locality from flattering either tree. + var random = std.Random.DefaultPrng.init(0xBB7EE); + random.random().shuffle([]const u8, queries.items); + for (queries.items) |q| { + const expected = baseline.search(q); + const actual = candidate.search(q); + if (!std.mem.eql(u8, std.mem.sliceAsBytes(expected), std.mem.sliceAsBytes(actual))) return error.ExactParityFailed; + const rank = candidate.bb_dictionary.?.lowerBound(q); + var lo: usize = 0; + var hi = baseline.word_dir.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + if (std.mem.order(u8, bb.word(data, baseline.word_dir[mid]), q) == .lt) lo = mid + 1 else hi = mid; + } + if (rank != lo) return error.LowerBoundParityFailed; + } + const prefix_count = @min(1000, queries.items.len); + for (queries.items[0..prefix_count]) |q| { + const expected = try baseline.searchPrefix(q, a, 100); + defer a.free(expected); + const actual = try candidate.searchPrefix(q, a, 100); + defer a.free(actual); + if (!std.mem.eql(u8, std.mem.sliceAsBytes(expected), std.mem.sliceAsBytes(actual))) return error.PrefixParityFailed; + } + std.debug.print("files={d} source_bytes={d} words={d} queries={d} exact/lower-bound parity=all prefix_parity={d} extra_bytes={d} build_ms={d:.3}\n", .{ paths.items.len, source_bytes, baseline.word_dir.len, queries.items.len, prefix_count, candidate.bb_dictionary.?.bytes, @as(f64, @floatFromInt(build_ns)) / 1e6 }); + // Parity walks pre-fault both mappings. Alternate AB/BA over seven rounds. + var checksum: usize = 0; + for (0..7) |round| { + for (0..2) |turn| { + const is_bb = (round + turn) % 2 != 0; + const index = if (is_bb) &candidate else &baseline; + var start = cio.nanoTimestamp(); + for (0..5) |_| for (queries.items) |q| { + checksum +%= index.search(q).len; + }; + const exact = @as(f64, @floatFromInt(cio.nanoTimestamp() - start)) / @as(f64, @floatFromInt(5 * queries.items.len)); + start = cio.nanoTimestamp(); + for (queries.items[0..prefix_count]) |q| { + const hits = try index.searchPrefix(q, a, 100); + checksum +%= hits.len; + a.free(hits); + } + const prefix = @as(f64, @floatFromInt(cio.nanoTimestamp() - start)) / @as(f64, @floatFromInt(prefix_count)); + std.debug.print("round={d} implementation={s} exact_ns={d:.2} prefix_ns={d:.2}\n", .{ round, if (is_bb) "bbtree" else "binary", exact, prefix }); + } + } + // Exercise tree teardown before mmap promotion and verify retained postings. + try candidate.indexFile("bbtree-promotion-fixture.zig", "zzbbpromotionfixturezz"); + if (candidate.bb_dictionary != null or candidate.mmap_data != null or candidate.search("zzbbpromotionfixturezz").len != 1) return error.PromotionFailed; + for (baseline.word_dir) |off| { + const key = bb.word(data, off); + if (!std.mem.eql(u8, std.mem.sliceAsBytes(baseline.search(key)), std.mem.sliceAsBytes(candidate.search(key)))) return error.PromotionParityFailed; + } + std.debug.print("promotion_parity=all checksum={d}\n", .{checksum}); +} diff --git a/src/index.zig b/src/index.zig index 76f4da09..0db98dd0 100644 --- a/src/index.zig +++ b/src/index.zig @@ -1,6 +1,7 @@ const std = @import("std"); const cio = @import("cio.zig"); const ContentCache = @import("hot_cache.zig").ContentCache; +const BBDictionary = @import("bbtree_dictionary.zig").Dictionary; // ── Inverted word index ───────────────────────────────────── // Maps word → list of (path, line) hits. O(1) word lookup. @@ -34,13 +35,22 @@ pub const WordIndex = struct { /// Sum of all values in doc_lengths. total_tokens: u64 = 0, /// Zero-copy read mode: when non-null, the postings live in this mmap'd - /// word.index file and `index` is empty — search/searchPrefix binary-search - /// the sorted on-disk words instead. The first write promotes to a heap - /// index (promoteIfBorrowed). Cuts warm-daemon RSS from ~4x the file to a + /// word.index file and `index` is empty — search/searchPrefix use a bounded + /// bbtree dictionary, with complete binary search as the low-memory fallback. + /// The first write promotes to a heap index (promoteIfBorrowed). + /// Cuts warm-daemon RSS from ~4x the file to a /// small word directory + touched pages. mmap_data: ?[]align(std.heap.page_size_min) const u8 = null, /// Byte offset of each sorted word record in mmap_data, for binary search. word_dir: []u32 = &.{}, + /// Default mmap accelerator; at most 8 MiB, with unchanged disk/posting data. + bb_dictionary: ?BBDictionary = null, + + pub fn enableBBDictionary(self: *WordIndex) !void { + if (self.bb_dictionary != null) return; + const data = self.mmap_data orelse return error.NotMmapBacked; + self.bb_dictionary = try BBDictionary.init(self.allocator, data, self.word_dir); + } pub fn hitPath(self: *const WordIndex, hit: WordHit) []const u8 { if (hit.doc_id < self.id_to_path.items.len) return self.id_to_path.items[hit.doc_id]; @@ -79,6 +89,7 @@ pub const WordIndex = struct { } pub fn deinit(self: *WordIndex) void { + if (self.bb_dictionary) |*dictionary| dictionary.deinit(); if (self.mmap_data) |m| { // Zero-copy mode: index/path_to_id/file_words are empty; id_to_path // owns the duped paths; word_dir + doc_lengths are heap; the postings @@ -452,6 +463,14 @@ pub const WordIndex = struct { /// offsets), hence align(1) — fields are read with unaligned loads. fn mmapSearch(self: *WordIndex, lower: []const u8) []align(1) const WordHit { const data = self.mmap_data.?; + if (self.bb_dictionary) |*dictionary| { + const rank = dictionary.find(lower) orelse return &.{}; + const off = self.word_dir[rank]; + const wlen = std.mem.readInt(u16, data[off..][0..2], .little); + const hc_off = off + 2 + wlen; + const hit_count = std.mem.readInt(u32, data[hc_off..][0..4], .little); + return std.mem.bytesAsSlice(WordHit, data[hc_off + 4 ..][0 .. @as(usize, hit_count) * @sizeOf(WordHit)]); + } var lo: usize = 0; var hi: usize = self.word_dir.len; while (lo < hi) { @@ -509,6 +528,8 @@ pub const WordIndex = struct { // Commit: drop the (empty) mmap-mode maps + unmap, install heap maps. self.index.deinit(); self.path_to_id.deinit(); + if (self.bb_dictionary) |*dictionary| dictionary.deinit(); + self.bb_dictionary = null; self.allocator.free(self.word_dir); cio.munmap(data); self.index = new_index; @@ -583,7 +604,10 @@ pub const WordIndex = struct { // Sorted on-disk words: binary-search the prefix lower bound, then // walk forward while the word still starts with `prefix`. var lo: usize = 0; - var hi: usize = self.word_dir.len; + var hi: usize = if (self.bb_dictionary) |*dictionary| blk: { + lo = dictionary.lowerBound(prefix); + break :blk lo; + } else self.word_dir.len; while (lo < hi) { const mid = lo + (hi - lo) / 2; const off = self.word_dir[mid]; @@ -918,7 +942,15 @@ pub const WordIndex = struct { /// WordIndex in zero-copy mode (first write promotes to heap). Null on any /// malformation — caller can fall back to readFromDisk. pub fn mmapFromDisk(io: std.Io, dir_path: []const u8, allocator: std.mem.Allocator) ?WordIndex { - return mmapFromDiskInner(io, dir_path, allocator) catch null; + return mmapFromDiskWithDictionary(io, dir_path, allocator, true); + } + + /// Explicit binary-search control for paired resource/correctness checks. + /// Failure to allocate the accelerator never drops words or postings. + pub fn mmapFromDiskWithDictionary(io: std.Io, dir_path: []const u8, allocator: std.mem.Allocator, enabled: bool) ?WordIndex { + var result = (mmapFromDiskInner(io, dir_path, allocator) catch return null) orelse return null; + if (enabled) result.enableBBDictionary() catch {}; + return result; } fn mmapFromDiskInner(io: std.Io, dir_path: []const u8, allocator: std.mem.Allocator) !?WordIndex { diff --git a/src/release_info.zig b/src/release_info.zig index 5d39db3d..6939bc06 100644 --- a/src/release_info.zig +++ b/src/release_info.zig @@ -1 +1 @@ -pub const semver = "0.2.5861"; +pub const semver = "0.2.5862"; diff --git a/src/test_bbtree.zig b/src/test_bbtree.zig new file mode 100644 index 00000000..e45e1a5a --- /dev/null +++ b/src/test_bbtree.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const bb = @import("bbtree_dictionary.zig"); +const a = std.testing.allocator; + +fn check(keys: []const []const u8) !void { + var bytes: std.ArrayList(u8) = .empty; + defer bytes.deinit(a); + const offsets = try a.alloc(u32, keys.len); + defer a.free(offsets); + for (keys, 0..) |key, i| { + offsets[i] = @intCast(bytes.items.len); + var length: [2]u8 = undefined; + std.mem.writeInt(u16, &length, @intCast(key.len), .little); + try bytes.appendSlice(a, &length); + try bytes.appendSlice(a, key); + } + var tree = try bb.Dictionary.init(a, bytes.items, offsets); + defer tree.deinit(); + try std.testing.expectEqual(tree.bytes, tree.storage.len); + try std.testing.expect(tree.bytes <= bb.Dictionary.max_bytes); + for (keys, 0..) |key, i| { + try std.testing.expectEqual(@as(?usize, i), tree.find(key)); + try std.testing.expectEqual(i, tree.lowerBound(key)); + const absent = try std.fmt.allocPrint(a, "{s}~", .{key}); + defer a.free(absent); + try std.testing.expectEqual(@as(?usize, null), tree.find(absent)); + var after = i + 1; + while (after < keys.len and std.mem.order(u8, keys[after], absent) == .lt) : (after += 1) {} + try std.testing.expectEqual(after, tree.lowerBound(absent)); + const prefix = key[0 .. key.len / 2]; + var expected: usize = 0; + while (expected < keys.len and std.mem.order(u8, keys[expected], prefix) == .lt) : (expected += 1) {} + try std.testing.expectEqual(expected, tree.lowerBound(prefix)); + } + try std.testing.expectEqual(@as(?usize, null), tree.find("")); + try std.testing.expectEqual(@as(usize, 0), tree.lowerBound("")); + try std.testing.expectEqual(@as(?usize, null), tree.find("\xff\xff")); + try std.testing.expectEqual(keys.len, tree.lowerBound("\xff\xff")); +} + +test "bbtree dictionary: empty, page boundaries and multilevel fence prefixes" { + for ([_]usize{ 0, 1, 29, 30, 31, 41, 42, 1300, 1301, 1302, 55000 }) |count| { + const keys = try a.alloc([]const u8, count); + defer a.free(keys); + defer for (keys) |key| a.free(key); + for (keys, 0..) |*key, i| key.* = try std.fmt.allocPrint(a, "common_prefix_{d:0>8}_suffix", .{i}); + try check(keys); + } +} + +test "bbtree dictionary: long prefixes, length-tag collisions and prefix keys" { + const keys = try a.alloc([]const u8, 150); + defer a.free(keys); + defer for (keys) |key| a.free(key); + const shared: [100]u8 = @splat('a'); + for (keys, 0..) |*key, i| { + const suffix: [300]u8 = @splat('z'); + key.* = try std.fmt.allocPrint(a, "{s}{d:0>8}{s}", .{ &shared, i, suffix[0 .. i * 2] }); + } + try check(keys); + // Long leaf-window ties, including differing lengths and saturated tags. + const tied = [_][]const u8{ "abcdefgh", "abcdefgha", "abcdefghaa", "abcdefghaaa", "abcdefghb", "abcdefghbb", "abcdefghc" }; + try check(&tied); +} + +test "bbtree dictionary: maximum disk word lengths do not overflow prefix skips" { + const shared: [65532]u8 = @splat('a'); + const keys = try a.alloc([]const u8, 64); + defer a.free(keys); + defer for (keys) |key| a.free(key); + for (keys, 0..) |*key, i| key.* = try std.fmt.allocPrint(a, "{s}{d:0>3}", .{ &shared, i }); + try check(keys); +} diff --git a/src/test_bbtree_memory.zig b/src/test_bbtree_memory.zig new file mode 100644 index 00000000..a542708c --- /dev/null +++ b/src/test_bbtree_memory.zig @@ -0,0 +1,26 @@ +const std = @import("std"); +const bb = @import("bbtree_dictionary.zig"); + +fn construct(allocator: std.mem.Allocator) !void { + const data = "\x05\x00alpha"; + var tree = try bb.Dictionary.init(allocator, data, &.{0}); + defer tree.deinit(); + try std.testing.expectEqual(@as(?usize, 0), tree.find("alpha")); + try std.testing.expectEqual(@as(usize, 1024), tree.storage.len); +} + +test "bbtree memory: single slab allocation failure releases everything" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, construct, .{}); +} + +test "bbtree memory: budget ceiling rejects before allocating" { + const data = "\x05\x00alpha"; + try std.testing.expectError(error.DictionaryBudgetExceeded, + bb.Dictionary.initWithBudget(std.testing.allocator, data, &.{0}, 1023)); + var tree = try bb.Dictionary.initWithBudget(std.testing.allocator, data, &.{0}, 1024); + defer tree.deinit(); + try std.testing.expectEqual(@as(usize, 1024), tree.bytes); + var empty = try bb.Dictionary.initWithBudget(std.testing.allocator, &.{}, &.{}, 0); + defer empty.deinit(); + try std.testing.expectEqual(@as(?usize, null), empty.find("alpha")); +} diff --git a/src/test_index.zig b/src/test_index.zig index d4df60b7..b3f48c88 100644 --- a/src/test_index.zig +++ b/src/test_index.zig @@ -3328,6 +3328,11 @@ test "mmap word index: zero-copy load matches heap load and promotes on write" { var mm = WordIndex.mmapFromDisk(io, dir, alloc).?; defer mm.deinit(); // picks the right path (mmap, or heap after promote) try testing.expect(mm.mmap_data != null); + try testing.expect(mm.bb_dictionary != null); + var binary = WordIndex.mmapFromDiskWithDictionary(io, dir, alloc, false).?; + defer binary.deinit(); + try testing.expect(binary.bb_dictionary == null); + try testing.expectEqualSlices(u8, std.mem.sliceAsBytes(binary.search("alphaToken")), std.mem.sliceAsBytes(mm.search("alphaToken"))); // search parity (exact) try testing.expectEqual(heap.search("alphaToken").len, mm.search("alphaToken").len); @@ -3362,6 +3367,7 @@ test "mmap word index: zero-copy load matches heap load and promotes on write" { // Promote on write: a mutation materializes a heap index, then stays queryable. try mm.indexFile("src/c.zig", "pub fn gammaToken() void {}\n"); try testing.expect(mm.mmap_data == null); + try testing.expect(mm.bb_dictionary == null); const g = try mm.searchDeduped("gammaToken", alloc); defer alloc.free(g); try testing.expectEqual(@as(usize, 1), g.len);