diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75eac..d2d363fb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -133,7 +133,7 @@ jobs: strategy: fail-fast: false matrix: - mode: [simulation, walltime] + mode: [simulation, walltime, memory] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/Cargo.lock b/Cargo.lock index 1acec08f4..fee8f75f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2161,6 +2161,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "libredox" version = "0.1.16" @@ -2399,9 +2408,11 @@ dependencies = [ "libbpf-rs", "libc", "log", + "mimalloc", "object", "parking_lot", "paste", + "perf-event-open-sys", "rayon", "rstest", "runner-shared", @@ -2437,6 +2448,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" @@ -2813,6 +2833,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perf-event-open-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5f8d1487a4ffa23c80a1c355dd27235f9b66fb71ba0f261eb417e4fe8451347" +dependencies = [ + "libc", +] + [[package]] name = "pest" version = "2.8.6" @@ -3616,6 +3645,7 @@ dependencies = [ "rmp", "rmp-serde", "serde", + "serde_bytes", "serde_json", "zstd", ] @@ -4062,6 +4092,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/Cargo.toml b/Cargo.toml index 8bd28f039..34be934d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ ipc-channel = "0.20" itertools = "0.14.0" rayon = "1.12" linux-perf-event-reader = "0.10.2" # matches the version linux-perf-data resolves to +perf-event-open-sys = "6.0" env_logger = "0.11.10" tempfile = "3.27.0" object = { version = "0.39", default-features = false, features = ["read_core", "elf"] } diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 0c8d86d41..2554c18a6 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -20,11 +20,20 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed. +Mapping collection (`src/perf_mappings.rs`) uses Linux's native per-CPU perf event stream, not an LSM/BPF availability gate. `PerfMappingPoller` opens a `PERF_TYPE_SOFTWARE` dummy event with `PERF_ATTR_INHERIT` and `PERF_ATTR_MMAP2` on every online CPU for the tracked process, mmaps a perf ring per CPU, and drains those rings on a poll thread. It keeps executable mappings with absolute paths from `PERF_RECORD_MMAP2` and emits the single artifact representation, `MemtrackEventKind::Mapping` inside `MemtrackArtifact.events`, carrying the mapping's pid/tid/timestamp/address/path/device/inode/file offset/length. Opening or enabling any perf event requires the host's perf permissions (for example an allowed `perf_event_paranoid` policy or `CAP_PERFMON`); a permission error is returned from `Tracker::spawn` rather than silently disabling mapping collection. Kernel `PERF_RECORD_LOST` records, ring overruns, and malformed records increment the shared mapping-loss counter. `Tracker::dropped_events_count()` includes that counter with BPF ring-buffer drops, and `codspeed-memtrack track` aborts when the total is non-zero because the artifact is incomplete.` + +### Event stream compatibility + +Session relies on Rust's declaration-order field drop: _poller, _stack_poller, then _perf_mapping_poller. The BPF event and stack pollers therefore disconnect, fully drain, and join before the perf poller is dropped. PerfMappingPoller buffers mapping records and emits them during shutdown, after ordinary allocation/RSS/stack events have reached encode_events; encode_events preserves input order, so Mapping records are a terminal suffix in the one artifact stream. + +This ordering is compatibility-critical. Mapping is a newer event variant; older stream consumers may treat the first unknown Mapping as EOF. Keeping it as the suffix lets those consumers process the complete memory timeline before stopping at that first unknown record. Do not reorder the poller fields or emit mapping records before shutdown. + > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking. ## Key Directories -- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `stacks/`, `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/stack_capture.bpf.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/perf_mappings.rs` — native per-CPU `PERF_RECORD_MMAP2` collector. - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`. - `tests/` — integration tests + `snapshots/` (insta). - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces). @@ -75,7 +84,7 @@ sudo -E cargo test --test c_tests -- --test-threads 1 - **Build toolchain:** `clang` + BTF/vmlinux headers, `libbpf-dev`, `zlib1g-dev`, `pkgconf`, `build-essential`; vendored libbpf also needs `autopoint`/`bison`/`flex`. - `vmlinux.h` is pinned to a specific git rev; `libbpf-rs` uses the `vendored` feature (dist links `libbpf-rs/static`). -Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate). +Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_MEMTRACK_TRACK_ALLOCATORS` (0/false disables), `CODSPEED_MEMTRACK_TRACK_PHYSICAL` (1 enables), `CODSPEED_MEMTRACK_CAPTURE_STACKS` (1 enables), `CODSPEED_MEMTRACK_STACK_BUDGET` (stack copy size in bytes, default 8192), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate). ## Testing & QA diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 4912802da..82dccd265 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -16,7 +16,7 @@ required-features = ["ebpf"] [features] default = ["ebpf"] -ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux"] +ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux", "dep:mimalloc"] [dependencies] anyhow = { workspace = true } @@ -34,9 +34,11 @@ itertools = { workspace = true } paste = "1.0.15" libbpf-rs = { version = "0.26", features = ["vendored"], optional = true } object = { workspace = true } +perf-event-open-sys = { workspace = true } rayon = "1.12" parking_lot = "0.12" typed-builder = "0.23.2" +mimalloc = { version = "0.1", optional = true } [build-dependencies] libbpf-cargo = { version = "0.26", optional = true } diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 9a4cc2387..8de96317f 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -9,6 +9,7 @@ BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ SEC(UPROBE_SEC) \ int uprobe_##name(struct pt_regs* ctx) { \ + stash_stack_hash(capture_stack(ctx)); \ return store_param(&name##_arg, arg_expr); \ } \ SEC(URETPROBE_SEC) \ @@ -17,6 +18,7 @@ if (!arg_ptr) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ __u64 ret_val = PT_REGS_RC(ctx); \ if (ret_val == 0) { \ return 0; \ @@ -32,6 +34,7 @@ if (arg0 == 0) { \ return 0; \ } \ + __u64 stack_hash = capture_stack(ctx); \ submit_block; \ } @@ -50,6 +53,8 @@ return 0; \ } \ \ + stash_stack_hash(capture_stack(ctx)); \ + \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY); \ @@ -63,6 +68,7 @@ if (!args) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ \ struct name##_args_t a = *args; \ bpf_map_delete_elem(&name##_args, &tid); \ @@ -77,20 +83,22 @@ submit_block; \ } -UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), + { return submit_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) +UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), - { return submit_calloc_event(arg0, ret_val); }) + { return submit_calloc_event(arg0, ret_val, stack_hash); }) UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx), - { return submit_realloc_event(arg1, ret_val, arg0); }) + { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); }) UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx), - { return submit_aligned_alloc_event(arg0, ret_val); }) + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) /* * posix_memalign(void** memptr, size_t alignment, size_t size) @@ -115,6 +123,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } + stash_stack_hash(capture_stack(ctx)); + struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); return 0; @@ -127,6 +137,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { if (!args) { return 0; } + __u64 stack_hash = take_stack_hash(); struct posix_memalign_args_t a = *args; bpf_map_delete_elem(&posix_memalign_args, &tid); @@ -140,7 +151,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - return submit_aligned_alloc_event(a.size, addr); + return submit_aligned_alloc_event(a.size, addr, stack_hash); } struct mmap_args { diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bf0677c93..470e532b0 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -15,6 +15,47 @@ #define EVENT_TYPE_RSS 12 #define EVENT_TYPE_RMAP 13 +/* Largest user-stack copy one definition can carry. Every capture reserves + * header + budget in the ring up front, so this bounds ring space held per + * in-flight capture, per-capture copy cost, and the verifier work per load + * (which scales with the frozen budget). The kernel itself allows records up + * to the ring size; this is a policy cap. */ +#define MEMTRACK_MAX_STACK_COPY (32 * 1024) + +/* Registers, indexed by the capturing architecture's DWARF register number + * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30, + * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline + * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just + * ip/sp/bp. */ +#define MEMTRACK_STACK_REGS 33 + +#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 +#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 +/* bpf_get_stackid() has several negative outcomes (no user callchain, + * hash-bucket collision, or no free bucket), so this counts only missing ids. */ +#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2 +#define MEMTRACK_STACK_COUNTER_TRUNCATED 3 +#define MEMTRACK_STACK_COUNTER_RING_FULL 4 +#define MEMTRACK_STACK_COUNTER_COUNT 5 + +struct stack_regs { + uint64_t reg[MEMTRACK_STACK_REGS]; +}; + +/* Fixed header followed by `copy_len` bytes read upward from `sp`. */ +struct stack_header { + uint64_t hash; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ + uint32_t pid; + uint32_t tid; + uint32_t copy_len; + uint8_t truncated; /* the copy hit the size cap */ + uint8_t _pad[3]; + struct stack_regs regs; +}; + /* Common header shared by all event types */ struct event_header { uint8_t event_type; /* See EVENT_TYPE_* constants above */ @@ -29,20 +70,23 @@ struct event { union { /* Allocation events (malloc, calloc, aligned_alloc) */ struct { - uint64_t addr; /* address returned */ - uint64_t size; /* size requested */ + uint64_t addr; /* address returned */ + uint64_t size; /* size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } alloc; /* Deallocation event (free) */ struct { - uint64_t addr; /* address to free */ + uint64_t addr; /* address to free */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } free; /* Reallocation event - includes both old and new addresses */ struct { - uint64_t old_addr; /* previous address (can be NULL) */ - uint64_t new_addr; /* new address returned */ - uint64_t size; /* new size requested */ + uint64_t old_addr; /* previous address (can be NULL) */ + uint64_t new_addr; /* new address returned */ + uint64_t size; /* new size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } realloc; /* Memory mapping events (mmap, munmap, brk) */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 5a8d6ff07..b405f572b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -11,6 +11,7 @@ #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" +#include "stack_capture.bpf.h" #include "utils/event_helpers.h" #include "utils/folio.h" #include "utils/map_helpers.h" diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h new file mode 100644 index 000000000..2519767b5 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -0,0 +1,210 @@ +#ifndef __STACK_CAPTURE_BPF_H__ +#define __STACK_CAPTURE_BPF_H__ + +#include "event.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* Emit raw stack bytes and registers once per hash for offline DWARF unwinding. + * Allocation events carry the hash; stackid provides the frame-pointer fallback. + * Hashes may repeat after stack data changes or LRU eviction. + */ + +const volatile __u8 capture_stacks_enabled = 0; +const volatile __u32 stack_copy_budget = 4096; + +#define STACK_TRACE_MAX_DEPTH 127 +#define STACK_COPY_CHUNK 512 +#define FNV64_OFFSET 0xcbf29ce484222325ULL +#define FNV64_PRIME 0x00000100000001b3ULL + +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 16384); + __type(key, __u32); + __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +/* A separate ring keeps allocation events fixed-size. */ +BPF_RINGBUF(stacks, 512 * 1024 * 1024); +BPF_LRU_HASH_MAP(seen_stack_hashes, __u64, __u8, 262144); +BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); +BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); + +static __always_inline void bump_stack_counter(__u32 index) { + __u64* slot = bpf_map_lookup_elem(&stack_counters, &index); + if (slot) { + __sync_fetch_and_add(slot, 1); + } +} + +/* 4-lane FNV-1a over one STACK_COPY_CHUNK worth of 8-byte words. Fixed-size, + * unrolled so the verifier sees a bounded loop. */ +static __always_inline void fnv64_hash_chunk(__u64 lanes[4], const __u64* words) { +#pragma unroll + for (__u32 i = 0; i < STACK_COPY_CHUNK / 8; i += 4) { + lanes[0] = (lanes[0] ^ words[i]) * FNV64_PRIME; + lanes[1] = (lanes[1] ^ words[i + 1]) * FNV64_PRIME; + lanes[2] = (lanes[2] ^ words[i + 2]) * FNV64_PRIME; + lanes[3] = (lanes[3] ^ words[i + 3]) * FNV64_PRIME; + } +} + +#if defined(__TARGET_ARCH_x86) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + out->reg[0] = ctx->ax; + out->reg[1] = ctx->dx; + out->reg[2] = ctx->cx; + out->reg[3] = ctx->bx; + out->reg[4] = ctx->si; + out->reg[5] = ctx->di; + out->reg[6] = ctx->bp; + out->reg[7] = ctx->sp; + out->reg[8] = ctx->r8; + out->reg[9] = ctx->r9; + out->reg[10] = ctx->r10; + out->reg[11] = ctx->r11; + out->reg[12] = ctx->r12; + out->reg[13] = ctx->r13; + out->reg[14] = ctx->r14; + out->reg[15] = ctx->r15; + out->reg[16] = ctx->ip; +} +#elif defined(__TARGET_ARCH_arm64) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + struct user_pt_regs* uregs = (struct user_pt_regs*)ctx; +#pragma unroll + for (int i = 0; i < 31; i++) { + out->reg[i] = uregs->regs[i]; + } + out->reg[31] = uregs->sp; + out->reg[32] = uregs->pc; +} +#else +#error "stack capture needs a DWARF register mapping for this architecture" +#endif + +static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { + void* slot = bpf_ringbuf_reserve(&stacks, sizeof(struct stack_header) + stack_copy_budget, 0); + if (!slot) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); + return 0; + } + + __u64 sp = PT_REGS_SP(ctx); + __u8* payload = (__u8*)slot + sizeof(struct stack_header); + __u64 lanes[4] = { + FNV64_OFFSET ^ 0, + FNV64_OFFSET ^ 1, + FNV64_OFFSET ^ 2, + FNV64_OFFSET ^ 3, + }; + __u32 got = 0; + + /* Chunked reads stop at the first unreadable stack region. + * Loop bound is checked against stack_copy_budget (a frozen rodata constant) + * so every slot access is provably in range. */ +#pragma clang loop unroll(disable) + for (__u32 off = 0; off + STACK_COPY_CHUNK <= stack_copy_budget; off += STACK_COPY_CHUNK) { + if (bpf_probe_read_user(payload + off, STACK_COPY_CHUNK, (void*)(sp + off)) != 0) { + break; + } + + fnv64_hash_chunk(lanes, (const __u64*)(payload + off)); + got = off + STACK_COPY_CHUNK; + } + + if (got == 0) { + bpf_ringbuf_discard(slot, 0); + bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); + return 0; + } + + __u8 truncated = got >= stack_copy_budget; + if (truncated) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); + } + + __u64 hash = + (((lanes[0] * FNV64_PRIME) ^ lanes[1]) * FNV64_PRIME ^ lanes[2]) * FNV64_PRIME ^ lanes[3]; + + /* Length distinguishes a full copy from the same bytes as a truncated prefix. + * Zero is reserved for allocation events without a stack. */ + hash = (hash ^ got) * FNV64_PRIME; + if (hash == 0) { + hash = FNV64_OFFSET; + } + + __u8 marker = 1; + long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); + if (gate_result == -17) { /* -EEXIST */ + bpf_ringbuf_discard(slot, 0); + return hash; + } + if (gate_result != 0) { + /* Re-emit when deduplication is full so the hash remains resolvable. */ + bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); + } + + __s64 stackid = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK); + if (stackid < 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_STACKID_FAILED); + } + + struct stack_header* header = (struct stack_header*)slot; + header->hash = hash; + header->timestamp = bpf_ktime_get_ns(); + header->stackid = stackid; + header->sp = sp; + header->pid = ids.tgid; + header->tid = ids.tid; + header->copy_len = got; + header->truncated = truncated; + header->_pad[0] = 0; + header->_pad[1] = 0; + header->_pad[2] = 0; + fill_stack_regs(&header->regs, ctx); + + bpf_ringbuf_submit(slot, 0); + return hash; +} + +static __always_inline __u64 capture_stack(struct pt_regs* ctx) { + if (!capture_stacks_enabled || !is_enabled()) { + return 0; + } + + struct task_ids ids = current_task_ids(); + if (!is_tracked(ids.tgid)) { + return 0; + } + + return capture_stack_inner(ctx, ids); +} + +static __always_inline void stash_stack_hash(__u64 hash) { + if (hash == 0) { + return; + } + + __u64 tid = current_tid(); + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); +} + +static __always_inline __u64 take_stack_hash(void) { + if (!capture_stacks_enabled) { + return 0; + } + + __u64 tid = current_tid(); + __u64* hash = bpf_map_lookup_elem(&pending_stack_hash, &tid); + if (!hash) { + return 0; + } + + __u64 value = *hash; + bpf_map_delete_elem(&pending_stack_hash, &tid); + return value; +} + +#endif /* __STACK_CAPTURE_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index ca5969a9a..ca53593d2 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -2,6 +2,7 @@ #define __EVENT_HELPERS_H__ #include "../event.h" +#include "../stack_capture.bpf.h" #include "map_helpers.h" #include "process_tracking.h" @@ -88,36 +89,44 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ } -static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); +static __always_inline int submit_free_event(__u64 addr, __u64 stack_hash) { + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { + e->data.free.addr = addr; + e->data.free.stack_hash = stack_hash; + }); } -static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { +static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, + __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; + e->data.realloc.stack_hash = stack_hash; }); } diff --git a/crates/memtrack/src/ebpf/c/utils/map_helpers.h b/crates/memtrack/src/ebpf/c/utils/map_helpers.h index 484fe9703..69422cdf9 100644 --- a/crates/memtrack/src/ebpf/c/utils/map_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/map_helpers.h @@ -9,6 +9,14 @@ __type(value, value_type); \ } name SEC(".maps") +#define BPF_LRU_HASH_MAP(name, key_type, value_type, max_ents) \ + struct { \ + __uint(type, BPF_MAP_TYPE_LRU_HASH); \ + __uint(max_entries, max_ents); \ + __type(key, key_type); \ + __type(value, value_type); \ + } name SEC(".maps") + #define BPF_ARRAY_MAP(name, value_type, max_ents) \ struct { \ __uint(type, BPF_MAP_TYPE_ARRAY); \ diff --git a/crates/memtrack/src/ebpf/c/utils/process_tracking.h b/crates/memtrack/src/ebpf/c/utils/process_tracking.h index 4d8444aaf..6c8a893bf 100644 --- a/crates/memtrack/src/ebpf/c/utils/process_tracking.h +++ b/crates/memtrack/src/ebpf/c/utils/process_tracking.h @@ -6,7 +6,7 @@ BPF_HASH_MAP(tracked_pids, __u32, __u8, 10000); BPF_HASH_MAP(pids_ppid, __u32, __u32, 10000); -BPF_ARRAY_MAP(tracking_enabled, __u8, 1); +__u8 tracking_enabled = 0; static __always_inline int is_tracked(__u32 pid) { if (bpf_map_lookup_elem(&tracked_pids, &pid)) { @@ -29,13 +29,7 @@ static __always_inline int is_tracked(__u32 pid) { } static __always_inline int is_enabled(void) { - __u32 key = 0; - __u8* enabled = bpf_map_lookup_elem(&tracking_enabled, &key); - /* ARRAY-map lookups can't fail for a valid index; fail closed if one ever does. */ - if (!enabled) { - return 0; - } - return *enabled; + return tracking_enabled; } static __always_inline void track_child(__u32 child_pid, __u32 parent_pid) { diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 4ed422a58..5a8abfce0 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -1,4 +1,6 @@ -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use crate::prelude::*; +use libbpf_rs::MapCore; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; // Include the bindings for event.h pub mod bindings { @@ -34,13 +36,20 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Malloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, + }, + ), + EVENT_TYPE_FREE => ( + event.data.free.addr, + MemtrackEventKind::Free { + stack_hash: event.data.free.stack_hash, }, ), - EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), EVENT_TYPE_CALLOC => ( event.data.alloc.addr, MemtrackEventKind::Calloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_REALLOC => ( @@ -48,12 +57,14 @@ pub fn parse_event(data: &[u8]) -> Option { MemtrackEventKind::Realloc { old_addr: Some(event.data.realloc.old_addr), size: event.data.realloc.size, + stack_hash: event.data.realloc.stack_hash, }, ), EVENT_TYPE_ALIGNED_ALLOC => ( event.data.alloc.addr, MemtrackEventKind::AlignedAlloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_MMAP => ( @@ -111,6 +122,74 @@ pub fn parse_event(data: &[u8]) -> Option { }) } +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { + warn!( + "malformed stack record: {} bytes, need at least {header_len}", + data.len() + ); + return None; + }; + + let record_len = header_len + header.copy_len as usize; + if data.len() < record_len { + warn!( + "malformed stack record: {} bytes, need {record_len}", + data.len() + ); + return None; + } + + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes: data[header_len..record_len].to_vec(), + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }), + }, + }; + + Some((event, header.stackid)) +} + +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + /// A request from the exec-mapping watcher to attach allocator probes. #[derive(Debug, Clone, Copy)] pub struct AttachRequest { @@ -157,6 +236,7 @@ mod tests { event.data.realloc.old_addr = 0x1000; event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; + event.data.realloc.stack_hash = 0xbeef; let bytes = event_bytes(&event); @@ -168,9 +248,14 @@ mod tests { assert_eq!(parsed.addr, 0x2000); match parsed.kind { - MemtrackEventKind::Realloc { old_addr, size } => { + MemtrackEventKind::Realloc { + old_addr, + size, + stack_hash, + } => { assert_eq!(old_addr, Some(0x1000)); assert_eq!(size, 256); + assert_eq!(stack_hash, 0xbeef); } _ => panic!("Expected Realloc event kind"), } @@ -186,6 +271,7 @@ mod tests { event.header.tid = 2000; event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; + event.data.alloc.stack_hash = 0x1234; let bytes = event_bytes(&event); @@ -197,8 +283,9 @@ mod tests { assert_eq!(parsed.addr, 0x1000); match parsed.kind { - MemtrackEventKind::Malloc { size } => { + MemtrackEventKind::Malloc { size, stack_hash } => { assert_eq!(size, 128); + assert_eq!(stack_hash, 0x1234); } _ => panic!("Expected Malloc event kind"), } @@ -277,3 +364,74 @@ mod tests { } } } + +#[cfg(test)] +mod stack_tests { + use super::*; + use crate::ebpf::events::bindings::stack_regs; + + fn encode(header: stack_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const stack_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + fn header(copy_len: u32) -> stack_header { + stack_header { + hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len, + truncated: 1, + _pad: [0; 3], + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); + let payload = [1, 2, 3, 4, 5]; + + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::Stack { record } = event.kind else { + panic!("expected Stack event"); + }; + + assert_eq!(record.hash, header.hash); + assert_eq!(record.sp, header.sp); + assert_eq!(record.regs, header.regs.reg.to_vec()); + assert_eq!(record.bytes, payload); + assert!(record.fp_chain.is_empty()); + assert!(record.truncated); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/fd_holder.rs b/crates/memtrack/src/ebpf/memtrack/fd_holder.rs new file mode 100644 index 000000000..cd594a581 --- /dev/null +++ b/crates/memtrack/src/ebpf/memtrack/fd_holder.rs @@ -0,0 +1,401 @@ +//! Fork children that hold duplicate fd references, so the terminal close +//! happens in a disposable process rather than here. +//! +//! Closing the last reference to a BPF link fd waits for an RCU-tasks-trace +//! grace period and can hang if the kernel is wedged. [`FdHolderSet`] +//! partitions the fds across children so those waits also run in parallel. + +use std::ops::Range; +use std::os::fd::RawFd; +use std::time::{Duration, Instant}; + +use crate::prelude::*; + +pub struct FdHolder { + child_pid: libc::pid_t, + write_fd: RawFd, +} + +/// Close every open fd except those listed in `keep` (must be sorted +/// ascending, e.g. via `sort_unstable`), by sweeping `close_range` over the +/// gaps between them. `close_range` silently ignores fds that are already +/// closed or out of range, so gaps may safely include fds we never opened. +/// +/// # Safety +/// Only safe to call in the single-threaded child right after `fork()`, +/// before any allocation, locking, or `Drop` impl runs — see +/// [`FdHolder::spawn`]. +unsafe fn close_fds_except(keep: &[RawFd]) { + let mut lo: u32 = 0; + for &fd in keep { + let fd = fd as u32; + if fd > lo { + // SAFETY: caller upholds the fork-child, no-allocation contract. + unsafe { + libc::close_range(lo, fd - 1, 0); + } + } + lo = fd.saturating_add(1); + } + // SAFETY: same as above. + unsafe { + libc::close_range(lo, u32::MAX, 0); + } +} + +impl FdHolder { + /// Fork a child that owns `all_fds[own]` once the caller drops its copies. + /// The child waits for a byte or EOF on a private pipe, then exits. + /// + /// `fork()` duplicates the whole fd table, so the child first closes + /// everything but its chunk and the pipe read end; otherwise it would keep + /// unrelated resources alive. It runs only async-signal-safe libc calls, + /// since forking a multithreaded process leaves locks and allocator state + /// unusable. + pub fn spawn(all_fds: &[RawFd], own: Range) -> std::io::Result { + let mut fds = [0i32; 2]; + // SAFETY: `fds` points to two valid `i32`s, as `pipe(2)` requires. + if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let [read_fd, write_fd] = fds; + + // Built in the parent, where allocation is still safe. `write_fd` is + // excluded on purpose: the child must give it up, and the sweep closes + // anything absent from this list. + let mut keep: Vec = Vec::with_capacity(all_fds[own.clone()].len() + 1); + keep.push(read_fd); + keep.extend_from_slice(&all_fds[own.clone()]); + keep.sort_unstable(); + + // SAFETY: `fork()` itself is always safe to call; the child branch + // below is restricted to async-signal-safe libc calls until `_exit`. + let pid = unsafe { libc::fork() }; + match pid { + -1 => { + let err = std::io::Error::last_os_error(); + // SAFETY: both fds were just opened by us above. + unsafe { + libc::close(read_fd); + libc::close(write_fd); + } + Err(err) + } + 0 => { + // Keep only the chunk and pipe read end; no Rust code after fork. + unsafe { + close_fds_except(&keep); + let mut buf = [0u8; 1]; + loop { + let n = libc::read(read_fd, buf.as_mut_ptr().cast(), buf.len()); + if n >= 0 { + break; + } + } + libc::_exit(0); + } + } + child_pid => { + // SAFETY: `read_fd` was just opened by us above. + unsafe { + libc::close(read_fd); + } + Ok(Self { + child_pid, + write_fd, + }) + } + } + } + + /// Tell the child to exit. Idempotent, and does not wait for the exit — + /// see [`Self::release`] and [`FdHolderSet::release_all`] for that. + fn signal_release(&mut self) { + if self.write_fd >= 0 { + // SAFETY: `write_fd` is our open pipe write fd. Writing a byte + // ensures the child's `read` returns immediately without + // depending on whether sibling children inherited `write_fd`. + unsafe { + let byte = 0u8; + libc::write(self.write_fd, (&byte as *const u8).cast(), 1); + libc::close(self.write_fd); + } + self.write_fd = -1; + } + } + + /// Signal the child and wait up to `timeout` for it to exit. + /// + /// `false` means the holder is abandoned: the kernel is still tearing down + /// its fds, and init reaps it once that finishes. + pub fn release(mut self, timeout: Duration) -> bool { + self.signal_release(); + + let deadline = Instant::now() + timeout; + loop { + let mut status = 0i32; + // SAFETY: `child_pid` is our own child; `status` is a valid + // out-pointer. `WNOHANG` never blocks. + let ret = unsafe { libc::waitpid(self.child_pid, &mut status, libc::WNOHANG) }; + if ret == self.child_pid { + return true; + } + if ret == -1 { + // ECHILD: nothing left to wait for, already reaped. Any + // other errno (notably EINTR) is transient — fall through + // and retry instead of reporting a false success. + if std::io::Error::last_os_error().raw_os_error() == Some(libc::ECHILD) { + return true; + } + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +impl Drop for FdHolder { + fn drop(&mut self) { + self.signal_release(); + } +} + +/// Forked holders for disjoint fd chunks. +pub struct FdHolderSet(Vec); + +impl FdHolderSet { + /// Fork up to `k` holders over roughly equal, contiguous chunks of `fds`. + /// + /// If a fork fails, release holders already created and return an empty set + /// so the caller falls back to direct teardown. + pub fn spawn(fds: &[RawFd], k: usize) -> Self { + if fds.is_empty() { + return Self(Vec::new()); + } + let k = k.clamp(1, fds.len()); + let chunk_len = fds.len().div_ceil(k); + + let mut holders = Vec::with_capacity(k); + for start in (0..fds.len()).step_by(chunk_len) { + let own = start..(start + chunk_len).min(fds.len()); + match FdHolder::spawn(fds, own) { + Ok(holder) => holders.push(holder), + Err(err) => { + debug!( + "Failed to fork fd holder child ({err:#}); falling back to a direct drop for all {} fds", + fds.len() + ); + for holder in holders { + holder.release(Duration::from_secs(5)); + } + return Self(Vec::new()); + } + } + } + Self(holders) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Signal every holder first, then poll them against one shared `timeout`, + /// instead of spending a separate budget on each. + /// + /// `false` means at least one holder is abandoned: the kernel is still + /// tearing down its fds, and init reaps them once that finishes. + pub fn release_all(mut self, timeout: Duration) -> bool { + for holder in &mut self.0 { + holder.signal_release(); + } + + let mut pending: Vec = self.0.iter().map(|holder| holder.child_pid).collect(); + let deadline = Instant::now() + timeout; + loop { + pending.retain(|&pid| { + let mut status = 0i32; + // SAFETY: `pid` is one of our own children; `status` is a + // valid out-pointer. `WNOHANG` never blocks. + let ret = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + if ret == pid { + return false; + } + if ret == -1 { + // ECHILD: nothing left to wait for. Any other errno + // (notably EINTR) is transient; keep polling instead of + // treating it as a reap. + return std::io::Error::last_os_error().raw_os_error() != Some(libc::ECHILD); + } + true + }); + if pending.is_empty() { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn holder_exits_only_after_release() { + let holder = FdHolder::spawn(&[], 0..0).unwrap(); + let pid = holder.child_pid; + + let mut status = 0i32; + // SAFETY: `pid` is our own child; `status` is a valid out-pointer. + let ret = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + assert_eq!(ret, 0, "holder exited before release() was called"); + + assert!( + holder.release(Duration::from_secs(5)), + "holder did not exit within the timeout after release()" + ); + + // SAFETY: same as above. + let ret = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + assert_eq!(ret, -1, "child pid was still waitable after being reaped"); + } + + /// Set a pipe read end non-blocking so a `read` on it reports "no data + /// yet" (`EAGAIN`) rather than blocking, letting the test distinguish + /// that from EOF (`read` returning `0`). + fn set_nonblocking(fd: RawFd) { + // SAFETY: `fd` is a valid, open fd owned by the caller. + unsafe { + let flags = libc::fcntl(fd, libc::F_GETFL); + libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK); + } + } + + /// `true` if `fd` currently reports EOF (every writer closed), `false` + /// if it reports "no data yet" (at least one writer still open). + fn is_eof(fd: RawFd) -> bool { + let mut buf = [0u8; 1]; + // SAFETY: `fd` is a valid, open, non-blocking pipe read end; `buf` + // is a valid 1-byte out-buffer. + let n = unsafe { libc::read(fd, buf.as_mut_ptr().cast(), buf.len()) }; + if n == 0 { + return true; + } + assert_eq!( + n, -1, + "expected EAGAIN (no data) or EOF (0), got {n} bytes of unexpected data" + ); + let errno = std::io::Error::last_os_error(); + assert_eq!( + errno.raw_os_error(), + Some(libc::EAGAIN), + "unexpected read error: {errno}" + ); + false + } + + /// Poll `fd` for EOF for up to `timeout`, to avoid a race between a + /// just-`fork`ed child's close sweep and this process's own check. + fn wait_for_eof(fd: RawFd, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + if is_eof(fd) { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(5)); + } + } + + /// The fd-ownership contract every consumer relies on: after a chunk's + /// fds are handed to a holder and the caller closes its own copies, + /// releasing one holder tears down only *its* chunk, leaving fds owned + /// by other holders untouched. + #[test] + fn release_only_tears_down_its_own_chunk() { + let mut pipe_a = [0i32; 2]; + let mut pipe_b = [0i32; 2]; + // SAFETY: both arrays point to two valid `i32`s, as `pipe(2)` + // requires. + unsafe { + assert_eq!(libc::pipe(pipe_a.as_mut_ptr()), 0); + assert_eq!(libc::pipe(pipe_b.as_mut_ptr()), 0); + } + let [read_a, write_a] = pipe_a; + let [read_b, write_b] = pipe_b; + set_nonblocking(read_a); + set_nonblocking(read_b); + + let fds = [write_a, write_b]; + let holder_a = FdHolder::spawn(&fds, 0..1).unwrap(); + let holder_b = FdHolder::spawn(&fds, 1..2).unwrap(); + + // SAFETY: both fds were just opened by us above. + unsafe { + libc::close(write_a); + libc::close(write_b); + } + + assert!(!is_eof(read_a), "holder_a should still hold write_a"); + assert!(!is_eof(read_b), "holder_b should still hold write_b"); + + assert!(holder_a.release(Duration::from_secs(5))); + assert!(is_eof(read_a), "releasing holder_a should close write_a"); + assert!( + !is_eof(read_b), + "releasing holder_a must not affect holder_b's write_b" + ); + + assert!(holder_b.release(Duration::from_secs(5))); + assert!(is_eof(read_b), "releasing holder_b should close write_b"); + + // SAFETY: our own read ends, still open. + unsafe { + libc::close(read_a); + libc::close(read_b); + } + } + + /// A holder must close inherited descriptors outside its assigned chunk, or + /// those descriptors can keep unrelated pipes or resources alive. + #[test] + fn holder_closes_fds_outside_its_chunk() { + let mut pipe_out = [0i32; 2]; + // SAFETY: `pipe_out` points to two valid `i32`s, as `pipe(2)` + // requires. + unsafe { + assert_eq!(libc::pipe(pipe_out.as_mut_ptr()), 0); + } + let [read_out, write_out] = pipe_out; + set_nonblocking(read_out); + + let holder = FdHolder::spawn(&[], 0..0).unwrap(); + + // SAFETY: `write_out` was just opened by us above. + unsafe { + libc::close(write_out); + } + + assert!( + wait_for_eof(read_out, Duration::from_secs(5)), + "holder kept an fd open that it was never given ownership of" + ); + + assert!(holder.release(Duration::from_secs(5))); + // SAFETY: our own read end, still open. + unsafe { + libc::close(read_out); + } + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index b1099abe0..dbfcaf276 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -85,8 +85,19 @@ macro_rules! attach_uprobe_uretprobe { let Some(offset) = symbols.offset(symbol) else { return Ok(false); }; + let key = (lib_path.to_path_buf(), offset); + if self.attached_offsets.contains(&key) { + log::trace!( + "Skipping alias {} at {:#x} in {} (already instrumented)", + symbol, + offset, + lib_path.display() + ); + return Ok(true); + } self.[](lib_path, offset) .with_context(|| format!("Failed to attach {symbol}"))?; + self.attached_offsets.insert(key); log::trace!("Attached {} at {:#x}", symbol, offset); Ok(true) } @@ -121,8 +132,19 @@ macro_rules! attach_uprobe { let Some(offset) = symbols.offset(symbol) else { return Ok(false); }; + let key = (lib_path.to_path_buf(), offset); + if self.attached_offsets.contains(&key) { + log::trace!( + "Skipping alias {} at {:#x} in {} (already instrumented)", + symbol, + offset, + lib_path.display() + ); + return Ok(true); + } self.[](lib_path, offset) .with_context(|| format!("Failed to attach {symbol}"))?; + self.attached_offsets.insert(key); log::trace!("Attached {} at {:#x}", symbol, offset); Ok(true) } diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index c7376d463..984b228a1 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -1,4 +1,5 @@ use super::MemtrackBpf; +use crate::ebpf::stacks::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; @@ -15,27 +16,23 @@ impl MemtrackBpf { } pub fn enable_tracking(&mut self) -> Result<()> { - let key = 0u32; - let value = true as u8; - with_skel!(self, skel => skel.maps.tracking_enabled.update( - &key.to_le_bytes(), - &value.to_le_bytes(), - libbpf_rs::MapFlags::ANY, - )) - .context("Failed to enable tracking")?; - Ok(()) + self.set_tracking(true) } pub fn disable_tracking(&mut self) -> Result<()> { - let key = 0u32; - let value = false as u8; - with_skel!(self, skel => skel.maps.tracking_enabled.update( - &key.to_le_bytes(), - &value.to_le_bytes(), - libbpf_rs::MapFlags::ANY, - )) - .context("Failed to disable tracking")?; - Ok(()) + self.set_tracking(false) + } + + fn set_tracking(&mut self, enabled: bool) -> Result<()> { + with_skel!(mut self, skel => { + let bss = skel + .maps + .bss_data + .as_deref_mut() + .context("bss map missing")?; + bss.tracking_enabled = enabled as u8; + Ok(()) + }) } /// Mark a (dev, ino) as classified so the watcher stops re-signalling for it. @@ -68,6 +65,10 @@ impl MemtrackBpf { ) } + pub fn stack_capture_stats(&self) -> Result { + StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) + } + pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 2ec1f04f5..ca29fde3a 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -1,13 +1,13 @@ +use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; use crate::prelude::*; use libbpf_rs::Link; use libbpf_rs::skel::OpenSkel; use libbpf_rs::skel::SkelBuilder; use std::collections::HashMap; use std::mem::MaybeUninit; +use std::os::fd::{AsFd, AsRawFd, RawFd}; use std::path::Path; -use crate::ebpf::poller::RingBufferPoller; - mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); } @@ -18,10 +18,10 @@ mod legacy { #[macro_use] mod macros; mod allocator; +mod fd_holder; mod maps; mod rmap; mod tracking; - pub use maps::OwnershipMaps; pub use rmap::RmapSupport; @@ -121,11 +121,17 @@ pub struct MemtrackBpf { pub(super) probes: Vec, rmap: RmapSupport, physical: bool, + /// `(lib_path, offset)` pairs already instrumented. glibc exports + /// symbols like `cfree` (and `free_sized`/`__libc_free` elsewhere) at + /// the same file offset as their canonical function; attaching each + /// alias would double-instrument the one underlying function. + attached_offsets: std::collections::HashSet<(std::path::PathBuf, usize)>, } impl MemtrackBpf { - /// Load the skeleton, defaulting to the variant a BPF token is available for. - pub fn load(options: TrackerOptions) -> Result { + pub fn new(options: &TrackerOptions) -> Result { + crate::kernel::KernelBtf::ensure_available()?; + let variant = options.variant.unwrap_or_else(|| { if has_delegated_bpf_token() { BpfVariant::Token @@ -134,8 +140,9 @@ impl MemtrackBpf { } }); let physical = options.physical; - crate::kernel::KernelBtf::ensure_available()?; - + let capture_stacks = options.stack_capture; + let stack_copy_budget = ((options.stack_budget / 512) * 512) + .clamp(512, crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY); let page_shift = page_shift()?; let rmap = if physical { RmapSupport::detect() @@ -162,6 +169,19 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } + if capture_stacks { + rodata.capture_stacks_enabled = 1; + rodata.stack_copy_budget = stack_copy_budget; + } + } + + // Avoid reserving the stack maps when capture is disabled. A + // ring buffer's size must stay a power-of-two page count. + if !capture_stacks { + open_skel.maps.stacks.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; } // Autoload is decided before load(), so missing fentry targets must be off here. @@ -172,7 +192,6 @@ impl MemtrackBpf { } }; } - // Mirrors the attach match in `tracking.rs`. match rmap { RmapSupport::Unsupported => { for_each_rmap_core_prog!(disable_rmap_prog); @@ -210,6 +229,7 @@ impl MemtrackBpf { probes: Vec::new(), rmap, physical, + attached_offsets: std::collections::HashSet::new(), }) } @@ -228,6 +248,39 @@ impl MemtrackBpf { )) } + /// Poll stack records and resolve their frame-pointer chains on a worker thread. + /// Map lookups are syscalls and must not stall the ring-buffer poller. + pub(crate) fn poll_stacks( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + use crate::ebpf::events; + use runner_shared::artifacts::MemtrackEventKind; + + // The resolver owns the map handle because it outlives this skeleton borrow. + let stack_traces = with_skel!(self, skel => { + libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) + .context("Failed to create handle for stack_traces map")? + }); + + let resolve = + move |(mut event, stackid): (runner_shared::artifacts::MemtrackEvent, i64)| { + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + event + }; + + with_skel!(self, skel => ThreadedRingBufferPoller::new( + &skel.maps.stacks, + events::parse_stack, + resolve, + tx, + poll_interval_ms, + )) + } + /// Poll the exec-mapping request ring buffer into `tx`. Same contract as /// [`Self::poll_events_with_channel`]. pub(crate) fn poll_attach_with_channel( @@ -248,27 +301,31 @@ impl MemtrackBpf { self.probes.len() } - /// Detach all BPF links in parallel. Closing a uprobe link blocks on two - /// RCU grace periods in the kernel, but concurrent waiters share grace - /// periods, so closing from many threads scales near-linearly. + /// Detach all BPF links without blocking on kernel link teardown: forked + /// holders own disjoint fd chunks and perform the terminal close in + /// parallel, while this process only drops its duplicate references. pub fn detach_probes(&mut self) { - const DETACH_THREADS: usize = 32; - - let mut probes = std::mem::take(&mut self.probes); + let probes = std::mem::take(&mut self.probes); if probes.is_empty() { return; } debug!("Detaching {} BPF links", probes.len()); let start = std::time::Instant::now(); - let chunk_size = probes.len().div_ceil(DETACH_THREADS); - std::thread::scope(|scope| { - while !probes.is_empty() { - let split_at = probes.len().saturating_sub(chunk_size); - let chunk = probes.split_off(split_at); - scope.spawn(move || drop(chunk)); - } - }); + + let fds: Vec = probes.iter().map(|p| p.as_fd().as_raw_fd()).collect(); + let holders = fd_holder::FdHolderSet::spawn(&fds, 32); + let holder_count = holders.len(); + + drop(probes); + + if !holders.is_empty() && !holders.release_all(std::time::Duration::from_secs(30)) { + warn!( + "Link teardown is stuck in the kernel; abandoning {holder_count} fd holder processes" + ); + return; + } + debug!("Detached BPF links in {:?}", start.elapsed()); } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 2aa96549d..6cd589ab9 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -4,9 +4,11 @@ mod memtrack; pub(crate) mod poller; mod proc_fs; mod spawn; +mod stacks; mod tracker; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; +pub use stacks::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/poller.rs b/crates/memtrack/src/ebpf/poller.rs index d80a15549..1e0a81e04 100644 --- a/crates/memtrack/src/ebpf/poller.rs +++ b/crates/memtrack/src/ebpf/poller.rs @@ -77,3 +77,58 @@ impl Drop for RingBufferPoller { } } } + +/// A [`RingBufferPoller`] whose parsed items need a further, potentially +/// expensive step (e.g. a BPF map lookup, which is a syscall) before they are +/// forwarded on `tx`. That step runs on a dedicated resolver thread instead +/// of the poll thread, so a slow per-record resolve can't make the poll +/// thread fall behind the ring and drop records. +pub struct ThreadedRingBufferPoller { + // Drop `ring` first: its poll thread drops the resolver's input sender. + // The resolver then drains parsed items and can be joined safely. + ring: Option, + resolver: Option>, +} + +impl ThreadedRingBufferPoller { + /// Poll `rb_map` with `parse` like [`RingBufferPoller::new`], but run + /// `resolve` on a separate thread: `parse` results are forwarded over an + /// internal channel, and `resolve` turns each one into the value sent on + /// `tx`. + pub fn new( + rb_map: &M, + parse: F, + resolve: R, + tx: Sender, + poll_interval_ms: u64, + ) -> Result + where + M: MapCore, + T: Send + 'static, + U: Send + 'static, + F: Fn(&[u8]) -> Option + Send + 'static, + R: Fn(T) -> U + Send + 'static, + { + let (parsed_tx, parsed_rx) = mpsc::channel::(); + let ring = RingBufferPoller::new(rb_map, parse, parsed_tx, poll_interval_ms)?; + let resolver = std::thread::spawn(move || { + for item in parsed_rx { + let _ = tx.send(resolve(item)); + } + }); + + Ok(Self { + ring: Some(ring), + resolver: Some(resolver), + }) + } +} + +impl Drop for ThreadedRingBufferPoller { + fn drop(&mut self) { + drop(self.ring.take()); + if let Some(resolver) = self.resolver.take() { + let _ = resolver.join(); + } + } +} diff --git a/crates/memtrack/src/ebpf/stacks.rs b/crates/memtrack/src/ebpf/stacks.rs new file mode 100644 index 000000000..88605294b --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks.rs @@ -0,0 +1,35 @@ +use crate::ebpf::events::bindings::*; +use crate::prelude::*; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct StackCaptureStats { + pub copy_failed: u64, + pub hash_map_full: u64, + pub stackid_failed: u64, + pub truncated: u64, + pub ring_full: u64, +} + +impl StackCaptureStats { + pub fn read(map: &impl libbpf_rs::MapCore) -> Result { + Ok(Self { + copy_failed: slot(map, MEMTRACK_STACK_COUNTER_COPY_FAILED)?, + hash_map_full: slot(map, MEMTRACK_STACK_COUNTER_HASH_MAP_FULL)?, + stackid_failed: slot(map, MEMTRACK_STACK_COUNTER_STACKID_FAILED)?, + truncated: slot(map, MEMTRACK_STACK_COUNTER_TRUNCATED)?, + ring_full: slot(map, MEMTRACK_STACK_COUNTER_RING_FULL)?, + }) + } +} + +fn slot(map: &impl libbpf_rs::MapCore, index: u32) -> Result { + let value = map + .lookup(&index.to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read stack counter {index}"))? + .ok_or_else(|| anyhow!("stack counter slot {index} missing"))?; + let bytes: [u8; 8] = value + .as_slice() + .try_into() + .map_err(|_| anyhow!("stack counter {index} has unexpected size"))?; + Ok(u64::from_ne_bytes(bytes)) +} diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 4841049c8..5221fdd0b 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,17 +1,23 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; +use crate::ebpf::stacks::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; #[derive(Debug, Clone, Copy, TypedBuilder)] pub struct TrackerOptions { + /// BPF attach mechanism, or automatic detection when unset. + #[builder(default)] + pub variant: Option, /// Attach allocator uprobes (malloc/free/calloc/...) through the /// exec-mapping watcher. #[builder(default = true)] @@ -20,9 +26,13 @@ pub struct TrackerOptions { /// folio rmap hooks, which only attach on kernels that expose them. #[builder(default = false)] pub physical: bool, - /// Uprobe attach mechanism. `None` detects it from BPF token availability. - #[builder(default, setter(strip_option))] - pub variant: Option, + /// Capture allocation call stacks, adding per-allocation stack walking and + /// raw stack copying. + #[builder(default = false)] + pub stack_capture: bool, + /// Maximum bytes of user stack to copy per captured call stack. + #[builder(default = 8192)] + pub stack_budget: u32, } impl TrackerOptions { @@ -33,14 +43,37 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .physical(std::env::var("CODSPEED_MEMTRACK_TRACK_PHYSICAL").is_ok_and(|v| v == "1")) + .stack_capture( + std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").is_ok_and(|v| v == "1"), + ) + .stack_budget( + std::env::var("CODSPEED_MEMTRACK_STACK_BUDGET") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8192), + ) .build() } } +impl Default for TrackerOptions { + fn default() -> Self { + Self::builder().build() + } +} + pub struct Tracker { bpf: Arc>, worker: Mutex>, - allocators: bool, + options: TrackerOptions, + /// Number of native perf mapping records lost due to ring-buffer overflow. + mapping_lost: Arc, +} + +fn kill_and_wait(child: &mut std::process::Child) { + // Cleanup is best effort so the setup error remains the returned error. + let _ = child.kill(); + let _ = child.wait(); } impl Tracker { @@ -51,9 +84,13 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { + let bpf = MemtrackBpf::new(&options)?; + Self::build(bpf, options) + } + + fn build(mut bpf: MemtrackBpf, options: TrackerOptions) -> Result { Self::bump_memlock_rlimit()?; - let mut bpf = MemtrackBpf::load(options)?; bpf.attach_tracepoints()?; if options.allocators { bpf.attach_exec_watcher()?; @@ -69,7 +106,8 @@ impl Tracker { Ok(Self { bpf, worker: Mutex::new(worker), - allocators: options.allocators, + options, + mapping_lost: Arc::new(AtomicU64::new(0)), }) } @@ -82,31 +120,60 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { + let capture_stacks = self.options.stack_capture; + let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); } - let child = spawn_stopped(&mut wrapped)?; + let mut child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - match self.worker.lock().as_ref() { - Some(worker) => worker.set_root_pid(pid), - // No watcher to arm means exec mappings would be missed. - None if self.allocators => bail!("tracker already finished"), - None => {} - } - let (tx, rx) = mpsc::channel(); - let poller = { - let mut bpf = self.bpf.lock(); - bpf.add_tracked_pid(pid)?; - bpf.poll_events_with_channel(10, tx)? + let setup = (|| -> Result<_> { + match self.worker.lock().as_ref() { + Some(worker) => worker.set_root_pid(pid), + // No watcher to arm means exec mappings would be missed. + None if self.options.allocators => bail!("tracker already finished"), + None => {} + } + + let (tx, rx) = mpsc::channel(); + let (poller, stack_poller) = { + let mut bpf = self.bpf.lock(); + bpf.add_tracked_pid(pid)?; + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx.clone())?, stack_poller) + }; + let perf_mapping_poller = capture_stacks + .then(|| PerfMappingPoller::start(pid, tx, self.mapping_lost.clone())) + .transpose()?; + + Ok((rx, poller, stack_poller, perf_mapping_poller)) + })(); + let (rx, poller, stack_poller, perf_mapping_poller) = match setup { + Ok(pollers) => pollers, + Err(error) => { + kill_and_wait(&mut child); + return Err(error); + } }; - resume(pid)?; - Ok(Session::new(child, rx, poller)) - } + if let Err(error) = resume(pid) { + kill_and_wait(&mut child); + return Err(error); + } + Ok(Session::new( + child, + rx, + poller, + stack_poller, + perf_mapping_poller, + )) + } /// Enable allocator-event tracking in the BPF program. Lifetime events /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids /// regardless of this toggle. @@ -122,7 +189,16 @@ impl Tracker { /// Number of events the kernel dropped because the ring buffer was full. /// A non-zero value means the resulting trace is incomplete. pub fn dropped_events_count(&self) -> Result { - self.bpf.lock().dropped_events_count() + Ok(self.bpf.lock().dropped_events_count()? + self.mapping_lost.load(Ordering::Relaxed)) + } + + /// Per-cause counts of stack captures that were skipped or truncated. + pub fn stack_capture_stats(&self) -> Result { + self.bpf.lock().stack_capture_stats() + } + + pub fn stack_capture_enabled(&self) -> bool { + self.options.stack_capture } /// Only meaningful while the BPF object is alive; teardown frees the maps. diff --git a/crates/memtrack/src/lib.rs b/crates/memtrack/src/lib.rs index ccd93399f..30cfcc99c 100644 --- a/crates/memtrack/src/lib.rs +++ b/crates/memtrack/src/lib.rs @@ -4,6 +4,8 @@ mod bpf_token; mod ebpf; mod ipc; mod kernel; +#[cfg(feature = "ebpf")] +mod perf_mappings; pub mod prelude; #[cfg(feature = "ebpf")] mod session; diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 283cff194..64951cf2a 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -1,3 +1,6 @@ +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + use clap::Parser; use ipc_channel::ipc; use memtrack::prelude::*; @@ -159,6 +162,13 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; + if tracker.stack_capture_enabled() { + let stats = tracker + .stack_capture_stats() + .context("Failed to read stack capture stats")?; + debug!("stack capture stats: {stats:?}"); + } + // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. diff --git a/crates/memtrack/src/perf_mappings.rs b/crates/memtrack/src/perf_mappings.rs new file mode 100644 index 000000000..45350dbf4 --- /dev/null +++ b/crates/memtrack/src/perf_mappings.rs @@ -0,0 +1,437 @@ +use crate::prelude::*; +use perf_event_open_sys::bindings::{ + PERF_COUNT_SW_DUMMY, PERF_FLAG_FD_CLOEXEC, PERF_RECORD_LOST, PERF_RECORD_MMAP2, + PERF_SAMPLE_TID, PERF_SAMPLE_TIME, PERF_TYPE_SOFTWARE, perf_event_attr, perf_event_header, + perf_event_mmap_page, +}; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::io; +use std::mem::size_of; +use std::os::fd::RawFd; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::thread::JoinHandle; +use std::time::Duration; + +const DATA_PAGES: usize = 64; + +struct PerfRing { + fd: RawFd, + mapping: *mut u8, + mapping_len: usize, + data_offset: usize, + data_size: usize, + enabled: bool, +} + +// The mapping is exclusively consumed by the poll thread. +unsafe impl Send for PerfRing {} + +impl PerfRing { + fn open(pid: libc::pid_t, cpu: u32, page_size: usize) -> Result { + let mapping_len = page_size + .checked_mul(DATA_PAGES + 1) + .context("perf ring mapping size overflow")?; + ensure!( + mapping_len >= size_of::(), + "perf ring mapping is smaller than its metadata page" + ); + let mut attr = perf_event_attr { + type_: PERF_TYPE_SOFTWARE, + size: size_of::() as u32, + config: PERF_COUNT_SW_DUMMY as u64, + sample_type: (PERF_SAMPLE_TID | PERF_SAMPLE_TIME) as u64, + // PERF_FORMAT_LOST cannot account for inherited child events from this + // parent fd, so PERF_RECORD_LOST remains the complete loss signal. + read_format: 0, + clockid: libc::CLOCK_MONOTONIC, + ..Default::default() + }; + attr.__bindgen_anon_2.wakeup_events = 1; + attr.set_disabled(1); + attr.set_inherit(1); + attr.set_mmap(1); + attr.set_sample_id_all(1); + attr.set_mmap2(1); + attr.set_use_clockid(1); + + let fd = unsafe { + perf_event_open_sys::perf_event_open( + &mut attr, + pid, + cpu as _, + -1, + PERF_FLAG_FD_CLOEXEC as _, + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()) + .with_context(|| format!("perf_event_open failed for pid {pid} on CPU {cpu}")); + } + + let mapping = unsafe { + libc::mmap( + ptr::null_mut(), + mapping_len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if mapping == libc::MAP_FAILED { + let error = io::Error::last_os_error(); + unsafe { libc::close(fd) }; + return Err(error).context("failed to mmap perf mapping-event ring buffer"); + } + + let page = unsafe { &*(mapping.cast::()) }; + let data_offset = match usize::try_from(page.data_offset) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data offset"); + } + }; + let data_size = match usize::try_from(page.data_size) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data size"); + } + }; + let ring = Self { + fd, + mapping: mapping.cast(), + mapping_len, + data_offset, + data_size, + enabled: false, + }; + + ensure!( + data_offset >= page_size && data_offset % page_size == 0, + "kernel returned an invalid perf ring data offset" + ); + ensure!( + data_size >= size_of::() + && data_size % page_size == 0 + && data_size.is_power_of_two(), + "kernel returned an invalid perf ring data size" + ); + let data_end = data_offset + .checked_add(data_size) + .context("perf ring data range overflow")?; + ensure!( + data_end <= mapping_len, + "kernel returned a perf ring outside the mapped area" + ); + + Ok(ring) + } + + fn enable(&mut self) -> Result<()> { + if unsafe { perf_event_open_sys::ioctls::ENABLE(self.fd, 0) } < 0 { + return Err(io::Error::last_os_error()).context("failed to enable perf mapping events"); + } + self.enabled = true; + Ok(()) + } + + fn drain(&mut self, mappings: &mut Vec, lost: &AtomicU64) { + let page = unsafe { &mut *(self.mapping.cast::()) }; + let head = unsafe { ptr::read_volatile(&page.data_head) }; + std::sync::atomic::fence(Ordering::Acquire); + let mut tail = unsafe { ptr::read_volatile(&page.data_tail) }; + let available = head.wrapping_sub(tail); + + // Once the producer has lapped the consumer, the beginning of the + // stream no longer has a record boundary. Skip the corrupt prefix and + // let the kernel's PERF_RECORD_LOST record account for normal overflow. + if available > self.data_size as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + } else { + while tail != head { + let available = head.wrapping_sub(tail); + if available < size_of::() as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let header = self.copy_from_ring(tail, size_of::()); + let size = u16::from_ne_bytes([header[6], header[7]]) as usize; + if !(size_of::()..=self.data_size).contains(&size) + || size as u64 > available + { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let record = self.copy_from_ring(tail, size); + self.handle_record(&record, mappings, lost); + tail = tail.wrapping_add(size as u64); + } + } + + std::sync::atomic::fence(Ordering::Release); + unsafe { ptr::write_volatile(&mut page.data_tail, tail) }; + } + + fn copy_from_ring(&self, offset: u64, len: usize) -> Vec { + debug_assert!(len <= self.data_size); + let start = offset as usize & (self.data_size - 1); + let first_len = len.min(self.data_size - start); + let data = unsafe { self.mapping.add(self.data_offset) }; + let mut out = Vec::with_capacity(len); + unsafe { + out.extend_from_slice(std::slice::from_raw_parts(data.add(start), first_len)); + if first_len < len { + out.extend_from_slice(std::slice::from_raw_parts(data, len - first_len)); + } + } + out + } + + fn handle_record(&self, record: &[u8], mappings: &mut Vec, lost: &AtomicU64) { + match read_u32(record, 0) { + Some(PERF_RECORD_MMAP2) => { + if let Some(event) = parse_mmap2(record) { + mappings.push(event); + } + } + Some(PERF_RECORD_LOST) => match read_u64(record, 16) { + Some(count) => { + lost.fetch_add(count, Ordering::Relaxed); + } + None => { + lost.fetch_add(1, Ordering::Relaxed); + } + }, + _ => {} + } + } +} + +impl Drop for PerfRing { + fn drop(&mut self) { + unsafe { + if self.enabled { + let _ = perf_event_open_sys::ioctls::DISABLE(self.fd, 0); + } + libc::munmap(self.mapping.cast(), self.mapping_len); + libc::close(self.fd); + } + } +} + +pub(crate) struct PerfMappingPoller { + ctl: Option>>, + thread: Option>, +} + +impl PerfMappingPoller { + pub(crate) fn start( + pid: libc::pid_t, + tx: Sender, + lost: Arc, + ) -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "failed to read the system page size"); + + let cpus = online_cpus()?; + ensure!(!cpus.is_empty(), "no online CPUs reported by the kernel"); + let mut rings = Vec::with_capacity(cpus.len()); + for cpu in cpus { + rings.push(PerfRing::open(pid, cpu, page_size as usize)?); + } + for ring in &mut rings { + ring.enable()?; + } + + let (ctl, ctl_rx) = mpsc::channel::>(); + let thread = std::thread::spawn(move || { + let mut mappings = Vec::new(); + loop { + match ctl_rx.recv_timeout(Duration::from_millis(10)) { + Ok(ack) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + let _ = ack.send(()); + } + Err(RecvTimeoutError::Timeout) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + } + Err(RecvTimeoutError::Disconnected) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + mappings.sort_unstable_by_key(|event| (event.pid, event.timestamp)); + for mapping in mappings { + let _ = tx.send(mapping); + } + break; + } + } + } + }); + + Ok(Self { + ctl: Some(ctl), + thread: Some(thread), + }) + } +} + +impl Drop for PerfMappingPoller { + fn drop(&mut self) { + drop(self.ctl.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn parse_mmap2(record: &[u8]) -> Option { + const FIXED_END: usize = 72; + const SAMPLE_ID_SIZE: usize = 16; + if record.len() < FIXED_END + SAMPLE_ID_SIZE + || read_u32(record, 0)? != PERF_RECORD_MMAP2 + || read_u16(record, 6)? as usize != record.len() + { + return None; + } + + let prot = read_u32(record, 64)?; + if prot & libc::PROT_EXEC as u32 == 0 { + return None; + } + + let path_end = record.len() - SAMPLE_ID_SIZE; + let path_bytes = &record[FIXED_END..path_end]; + let nul = path_bytes.iter().position(|byte| *byte == 0)?; + let path = std::str::from_utf8(&path_bytes[..nul]).ok()?; + if !path.starts_with('/') { + return None; + } + + let major = read_u32(record, 40)? as u64; + let minor = read_u32(record, 44)? as u64; + Some(MemtrackEvent { + pid: read_u32(record, 8)? as libc::pid_t, + tid: read_u32(record, 12)? as libc::pid_t, + timestamp: read_u64(record, record.len() - 8)?, + addr: read_u64(record, 16)?, + kind: MemtrackEventKind::Mapping { + path: path.to_owned(), + dev: (major << 20) | minor, + ino: read_u64(record, 48)?, + file_offset: read_u64(record, 32)?, + len: read_u64(record, 24)?, + }, + }) +} + +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_ne_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_ne_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_ne_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +fn online_cpus() -> Result> { + let spec = std::fs::read_to_string("/sys/devices/system/cpu/online") + .context("failed to read online CPUs")?; + parse_cpu_list(spec.trim()) +} + +fn parse_cpu_list(spec: &str) -> Result> { + let mut cpus = Vec::new(); + for part in spec.split(',') { + let part = part.trim(); + ensure!(!part.is_empty(), "invalid empty CPU range"); + let (start, end) = match part.split_once('-') { + Some((start, end)) => (start.parse::()?, end.parse::()?), + None => { + let cpu = part.parse::()?; + (cpu, cpu) + } + }; + ensure!(start <= end, "invalid CPU range {part}"); + cpus.extend(start..=end); + } + Ok(cpus) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_cpu_ranges() { + assert_eq!(parse_cpu_list("0-2,5,8-9").unwrap(), vec![0, 1, 2, 5, 8, 9]); + } + + #[test] + fn parses_executable_mmap2() { + let path = b"/tmp/module.so\0"; + let mut record = vec![0; 72 + path.len() + 16]; + record[0..4].copy_from_slice(&PERF_RECORD_MMAP2.to_ne_bytes()); + let size = record.len() as u16; + record[6..8].copy_from_slice(&size.to_ne_bytes()); + record[8..12].copy_from_slice(&7_u32.to_ne_bytes()); + record[12..16].copy_from_slice(&8_u32.to_ne_bytes()); + record[16..24].copy_from_slice(&0x4000_u64.to_ne_bytes()); + record[24..32].copy_from_slice(&0x2000_u64.to_ne_bytes()); + record[32..40].copy_from_slice(&0x1000_u64.to_ne_bytes()); + record[40..44].copy_from_slice(&1_u32.to_ne_bytes()); + record[44..48].copy_from_slice(&2_u32.to_ne_bytes()); + record[48..56].copy_from_slice(&42_u64.to_ne_bytes()); + record[64..68].copy_from_slice(&(libc::PROT_EXEC as u32).to_ne_bytes()); + record[72..72 + path.len()].copy_from_slice(path); + let timestamp = 99_u64; + let time_offset = record.len() - 8; + record[time_offset..].copy_from_slice(×tamp.to_ne_bytes()); + + assert_eq!( + parse_mmap2(&record), + Some(MemtrackEvent { + pid: 7, + tid: 8, + timestamp, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: "/tmp/module.so".into(), + dev: (1 << 20) | 2, + ino: 42, + file_offset: 0x1000, + len: 0x2000, + }, + }) + ); + } +} diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 7aec33feb..53f02a50a 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -1,4 +1,5 @@ -use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; use std::process::{Child, ExitStatus}; @@ -9,7 +10,15 @@ use std::sync::mpsc::Receiver; pub struct Session { child: Child, events: Option>, + + // Drop order is part of the artifact compatibility contract. Rust drops + // fields in declaration order: both BPF pollers must stay before the perf + // mapping poller. Their Drop implementations disconnect, fully drain, and + // join their poll threads before PerfMappingPoller drops and emits its + // buffered Mapping records as the terminal stream suffix. _poller: RingBufferPoller, + _stack_poller: Option, + _perf_mapping_poller: Option, } impl Session { @@ -17,11 +26,15 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, + stack_poller: Option, + perf_mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, + _stack_poller: stack_poller, + _perf_mapping_poller: perf_mapping_poller, } } diff --git a/crates/memtrack/testdata/nested_doubling.c b/crates/memtrack/testdata/nested_doubling.c new file mode 100644 index 000000000..57badf398 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Each level allocates twice as much as its caller, then frees on the way back + * up, so the free order is the reverse of the allocation order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(1024) <-- free(2048) <-- free(4096) + * + * Every malloc and every free sits at a distinct call depth, so the six events + * also carry six distinct allocation stacks. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void) { + void* p = malloc(4096); + sleep(1); + escaped_pointer = p; + free(p); +} + +__attribute__((noinline)) static void level2(void) { + void* p = malloc(2048); + escaped_pointer = p; + sleep(1); + level3(); + free(p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + sleep(1); + level2(); + free(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/testdata/nested_doubling_shared_free.c b/crates/memtrack/testdata/nested_doubling_shared_free.c new file mode 100644 index 000000000..66c53fd99 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling_shared_free.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Same doubling allocation chain as nested_doubling.c, but ownership is handed + * down and the innermost level frees all three buffers in reverse order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(4096) + * free(2048) + * free(1024) + * + * The three mallocs come from three different call depths while all three frees + * share one, so a deallocation event must be attributed to the free site rather + * than to wherever its allocation happened. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void* outer, void* middle) { + void* p = malloc(4096); + escaped_pointer = p; + free(p); + free(middle); + free(outer); +} + +__attribute__((noinline)) static void level2(void* outer) { + void* p = malloc(2048); + escaped_pointer = p; + level3(outer, p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + level2(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/testdata/stack_paths.c b/crates/memtrack/testdata/stack_paths.c new file mode 100644 index 000000000..0cee7f437 --- /dev/null +++ b/crates/memtrack/testdata/stack_paths.c @@ -0,0 +1,46 @@ +#include +#include + +static volatile void *escaped_pointer; +static volatile unsigned int remaining_a = 50; +static volatile unsigned int remaining_b = 50; + +__attribute__((noinline)) static void path_a_inner(void) { + void *pointer = malloc(64); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_a(void) { + while (remaining_a != 0) { + path_a_inner(); + --remaining_a; + } +} + +__attribute__((noinline)) static void path_b_inner(void) { + void *pointer = malloc(192); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_b(void) { + while (remaining_b != 0) { + path_b_inner(); + --remaining_b; + } +} + +int main(void) { + void *marker_before = malloc(0xC0D59EED); + escaped_pointer = marker_before; + free(marker_before); + + path_a(); + path_b(); + + void *marker_after = malloc(0xC0D59EED); + escaped_pointer = marker_after; + free(marker_after); + return 0; +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index db5fe6439..f74de0530 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -2,14 +2,10 @@ mod shared; use rstest::rstest; +use shared::AllocationTestCase; use std::process::Command; use tempfile::TempDir; -struct AllocationTestCase { - name: &'static str, - source: &'static str, -} - const ALLOCATION_TEST_CASES: &[AllocationTestCase] = &[ AllocationTestCase { name: "double_malloc", @@ -89,7 +85,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box Result<(), Box Result<(), Box> { let malloc_addrs: HashSet = events .iter() .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size: 4242 } => Some(e.addr), + MemtrackEventKind::Malloc { size: 4242, .. } => Some(e.addr), _ => None, }) .collect(); let malloc_count = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let free_count = events .iter() .filter(|e| { - matches!(e.kind, MemtrackEventKind::Free) && malloc_addrs.contains(&e.addr) + matches!(e.kind, MemtrackEventKind::Free { .. }) + && malloc_addrs.contains(&e.addr) }) .count(); @@ -125,11 +126,11 @@ fn test_thread_dlopen() -> Result<(), Box> { |events| { let m4242 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let m4243 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243, .. })) .count(); assert_eq!(m4242, 100, "expected 100 mi_malloc(4242) events"); diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs index 9ca76196e..cdad60e44 100644 --- a/crates/memtrack/tests/rss_tests.rs +++ b/crates/memtrack/tests/rss_tests.rs @@ -2,6 +2,7 @@ mod shared; use itertools::Itertools; +use memtrack::TrackerOptions; use rstest::rstest; use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; use serde::Serialize; @@ -205,6 +206,13 @@ fn build_fixture( Ok((temp_dir, report_path, command)) } +fn rmap_only_options() -> TrackerOptions { + TrackerOptions::builder() + .allocators(false) + .physical(true) + .build() +} + /// Run a fixture under `track` and return the raw `/proc` RSS report it wrote to /// its argv[1] alongside the collected events. /// @@ -349,7 +357,9 @@ fn test_rss_rmap_tracking( #[case] source: &str, #[case] name: &str, ) -> Result<(), Box> { - let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let (raw_report, events) = track_fixture(source, name, |command| { + shared::track_command(command, rmap_only_options()) + })?; let raw_report = raw_report.ok_or("fixture wrote no rss report")?; let (rss_stat, rmap) = per_pid_peaks(&events); let summary = RssSummary { @@ -425,10 +435,14 @@ enum Reclaim { #[case::rss_stat(Reclaim::RssStat)] #[case::rmap(Reclaim::Rmap)] fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box> { + let options = match mode { + Reclaim::RssStat => TrackerOptions::default(), + Reclaim::Rmap => rmap_only_options(), + }; let (_report, events) = track_fixture( include_str!("../testdata/rss/madvise_extern.c"), "madvise_extern", - shared::track_command_with_rmap, + |command| shared::track_command(command, options), )?; // A = owner that faulted the file region; B = external caller, single-threaded @@ -597,7 +611,7 @@ fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box Result<(), Box, std::thread::JoinHandle<()>)>; +pub struct AllocationTestCase { + pub name: &'static str, + pub source: &'static str, +} + /// Snapshot every tracked event, ordered by timestamp and deduplicated by /// `(addr, kind)` so repeated tracking of one allocation counts once. /// @@ -31,7 +36,7 @@ macro_rules! assert_events_snapshot { matches!( e.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } @@ -90,11 +95,14 @@ macro_rules! assert_events_with_marker_for_each_variant { }; } -/// An event's kind and size, without the addresses that differ between runs of -/// the same workload. `Realloc` needs spelling out since its `Debug` includes the -/// old address. +/// An event's kind and size, without the addresses or stack identities that +/// differ between runs of the same workload. pub fn describe_kind(kind: &MemtrackEventKind) -> String { match kind { + MemtrackEventKind::Free { .. } => "Free".to_string(), + MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), + MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), + MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), MemtrackEventKind::Realloc { size, .. } => format!("Realloc {{ size: {size} }}"), other => format!("{other:?}"), } @@ -108,7 +116,7 @@ pub fn between_markers(events: &[Event]) -> Vec { const MARKER: u64 = 0xC0D5_9EED; let is_marker = - |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size } if size == MARKER); + |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size, .. } if size == MARKER); events .iter() @@ -124,6 +132,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -173,7 +182,11 @@ pub fn compile_rust_binary( /// Track a binary, collecting all memory events. pub fn track_binary(binary: &Path) -> TrackResult { - track_command(Command::new(binary)) + track_command(Command::new(binary), None) +} + +pub fn track_binary_with_env(binary: &Path) -> TrackResult { + track_command_with_tracker(Command::new(binary), Tracker::new()?) } pub fn compile_c_source( @@ -197,15 +210,9 @@ pub fn compile_c_source( Ok(binary_path) } -/// Track a command with the default probes: allocators only, discovered by the -/// exec-mapping watcher as the tracked tree maps executables. -pub fn track_command(command: Command) -> TrackResult { - track_command_with_opts(command, TrackerOptions::builder().build()) -} - -/// Track a command under a specific BPF variant rather than the detected one. -pub fn track_command_with_variant(command: Command, variant: BpfVariant) -> TrackResult { - track_command_with_opts(command, TrackerOptions::builder().variant(variant).build()) +pub fn track_command(command: Command, options: impl Into>) -> TrackResult { + let tracker = Tracker::with_options(options.into().unwrap_or_default())?; + track_command_with_tracker(command, tracker) } /// Physical-memory tracking without allocator probes. @@ -216,16 +223,6 @@ fn physical_only_options() -> TrackerOptions { .build() } -/// Track a command with physical-memory tracking enabled. -pub fn track_command_with_rmap(command: Command) -> TrackResult { - track_command_with_opts(command, physical_only_options()) -} - -/// Track a command with an explicit probe selection rather than the environment's. -pub fn track_command_with_opts(command: Command, options: TrackerOptions) -> TrackResult { - track_command_with_tracker(command, Tracker::with_options(options)?) -} - /// Track a command with rmap hooks and snapshot its ownership maps after the /// tracked tree exits but before tracker teardown frees the BPF maps. pub fn track_command_with_rmap_maps( @@ -281,7 +278,7 @@ fn event_profile(events: &[Event]) -> EventProfile { if !matches!( event.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } @@ -307,14 +304,14 @@ pub fn for_each_variant( let mut profiles: Vec<(BpfVariant, EventProfile)> = Vec::new(); for variant in [BpfVariant::Legacy, BpfVariant::Token] { - let tracker = - match Tracker::with_options(TrackerOptions::builder().variant(variant).build()) { - Ok(tracker) => tracker, - Err(err) => { - eprintln!("skipping {variant:?} variant, cannot attach here: {err:#}"); - continue; - } - }; + let options = TrackerOptions::builder().variant(Some(variant)).build(); + let tracker = match Tracker::with_options(options) { + Ok(tracker) => tracker, + Err(err) => { + eprintln!("skipping {variant:?} variant, cannot attach here: {err:#}"); + continue; + } + }; let (events, thread_handle) = track_command_with_tracker(workload(), tracker)?; assert_events(&events); diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap new file mode 100644 index 000000000..2bd6a37ce --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: false }", + "Malloc { size: 2048, has_stack: false }", + "Malloc { size: 4096, has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap new file mode 100644 index 000000000..2bd6a37ce --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: false }", + "Malloc { size: 2048, has_stack: false }", + "Malloc { size: 4096, has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap new file mode 100644 index 000000000..7cb31d31e --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_paths_stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths_stack_capture_disabled.snap new file mode 100644 index 000000000..cf9fc935d --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_paths_stack_capture_disabled.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/stack_budget_tests.rs b/crates/memtrack/tests/stack_budget_tests.rs new file mode 100644 index 000000000..d5f80ba9a --- /dev/null +++ b/crates/memtrack/tests/stack_budget_tests.rs @@ -0,0 +1,22 @@ +//! The stack copy budget is a frozen rodata constant, so the verifier's cost of +//! the capture program scales with it. Loading at the default proves nothing +//! about the maximum; both must load. +use memtrack::{BpfVariant, MemtrackBpf, TrackerOptions}; +use rstest::rstest; + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(8192)] +#[case(u32::MAX)] +#[test_log::test] +fn skeleton_loads_at_stack_budget(#[case] budget: u32) { + for variant in [BpfVariant::Legacy, BpfVariant::Token] { + let options = TrackerOptions::builder() + .variant(Some(variant)) + .stack_budget(budget) + .build(); + MemtrackBpf::new(&options).unwrap_or_else(|e| { + panic!("{variant:?} skeleton failed to load at budget {budget}: {e:#}") + }); + } +} diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs new file mode 100644 index 000000000..bc6ee76b3 --- /dev/null +++ b/crates/memtrack/tests/stack_tests.rs @@ -0,0 +1,139 @@ +#[macro_use] +mod shared; + +use itertools::Itertools; +use memtrack::TrackerOptions; +use rstest::rstest; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use shared::AllocationTestCase; +use std::mem::discriminant; +use std::process::Command; +use tempfile::TempDir; + +fn describe_allocator_event(kind: &MemtrackEventKind) -> Option { + let description = match kind { + MemtrackEventKind::Malloc { size, stack_hash } => { + format!("Malloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::Calloc { size, stack_hash } => { + format!("Calloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::AlignedAlloc { size, stack_hash } => format!( + "AlignedAlloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ), + MemtrackEventKind::Realloc { + size, stack_hash, .. + } => { + format!( + "Realloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ) + } + MemtrackEventKind::Free { stack_hash } => { + format!("Free {{ has_stack: {} }}", *stack_hash != 0) + } + _ => return None, + }; + + Some(description) +} + +fn format_events(events: &[MemtrackEvent]) -> Vec { + const MARKER: u64 = 0xC0D5_9EED; + let has_markers = events.iter().any(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { size, .. } if size == MARKER + ) + }); + + let filtered_events = if has_markers { + shared::between_markers(events) + } else { + events + .iter() + .filter(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { .. } + | MemtrackEventKind::Free { .. } + | MemtrackEventKind::Calloc { .. } + | MemtrackEventKind::Realloc { .. } + | MemtrackEventKind::AlignedAlloc { .. } + ) + }) + .sorted_by_key(|e| e.timestamp) + .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) + .cloned() + .collect() + }; + + filtered_events + .iter() + .filter_map(|e| describe_allocator_event(&e.kind)) + .collect() +} + +const STACK_TEST_CASES: &[AllocationTestCase] = &[ + AllocationTestCase { + name: "stack_paths", + source: include_str!("../testdata/stack_paths.c"), + }, + AllocationTestCase { + name: "nested_doubling", + source: include_str!("../testdata/nested_doubling.c"), + }, + AllocationTestCase { + name: "nested_doubling_shared_free", + source: include_str!("../testdata/nested_doubling_shared_free.c"), + }, +]; + +fn assert_stack_snapshot( + test_case: &AllocationTestCase, + stack_capture: bool, + snapshot_name: &str, +) -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source(test_case.source, test_case.name, temp_dir.path())?; + let options = TrackerOptions::builder() + .stack_capture(stack_capture) + .build(); + let (events, thread_handle) = shared::track_command(Command::new(binary), options)?; + + insta::assert_debug_snapshot!(snapshot_name, format_events(&events)); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(&STACK_TEST_CASES[0])] +#[case(&STACK_TEST_CASES[1])] +#[case(&STACK_TEST_CASES[2])] +#[test_log::test] +fn test_stack_capture( + #[case] test_case: &AllocationTestCase, +) -> Result<(), Box> { + assert_stack_snapshot(test_case, true, test_case.name) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(&STACK_TEST_CASES[0])] +#[case(&STACK_TEST_CASES[1])] +#[case(&STACK_TEST_CASES[2])] +#[test_log::test] +fn test_stack_capture_disabled( + #[case] test_case: &AllocationTestCase, +) -> Result<(), Box> { + assert_stack_snapshot( + test_case, + false, + &format!("{}_stack_capture_disabled", test_case.name), + ) +} diff --git a/crates/runner-shared/Cargo.toml b/crates/runner-shared/Cargo.toml index 8b8f6ab97..aee1707f0 100644 --- a/crates/runner-shared/Cargo.toml +++ b/crates/runner-shared/Cargo.toml @@ -4,9 +4,14 @@ publish = false version = "0.1.0" edition = "2024" +# Set by `cargo codspeed build` for the whole build; benches branch on it. +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(codspeed)'] } + [dependencies] anyhow = { workspace = true } serde = { workspace = true } +serde_bytes = "0.11" serde_json = { workspace = true } # Pinned to 1.x: 2.0 changes the wire format and serde integration bincode = "1.3" diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index a6c610e8e..1479fe96c 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -1,7 +1,22 @@ use divan::Bencher; +use divan::counter::{BytesCount, ItemsCount}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, MemtrackWriter, encode_events}; +use runner_shared::artifacts::{ + MemtrackEvent, MemtrackEventKind, MemtrackWriter, StackRecord, encode_events, +}; + +/// Reports allocation counts and bytes next to the timings for local runs. Only +/// tallies the thread running the benchmark, so the parallel encoder's +/// per-worker allocations show up in the single-threaded writer benches +/// instead. +/// +/// Left out of CodSpeed builds: wrapping the allocator costs ~15% on +/// allocation-heavy benchmarks, and the memory instrument reports allocations +/// there anyway. +#[cfg(not(codspeed))] +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); fn main() { divan::main(); @@ -14,14 +29,24 @@ fn generate_events(n: usize) -> Vec { for _ in 0..n { let size = rng.gen_range(8..8192); let kind = match rng.gen_range(0..10) { - 0 => MemtrackEventKind::Malloc { size }, - 1 => MemtrackEventKind::Free, + 0 => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, + 1 => MemtrackEventKind::Free { stack_hash: 0 }, 2 => MemtrackEventKind::Realloc { old_addr: Some(rng.r#gen()), size, + stack_hash: 0, + }, + 3 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, + 4 => MemtrackEventKind::AlignedAlloc { + size, + stack_hash: 0, }, - 3 => MemtrackEventKind::Calloc { size }, - 4 => MemtrackEventKind::AlignedAlloc { size }, 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, @@ -48,18 +73,25 @@ fn generate_events(n: usize) -> Vec { events } -#[divan::bench(args = [10_000, 100_000, 500_000, 1_000_000])] +/// Throughput of the single-threaded writer path: one zstd frame, no pool. +/// This is the per-worker ceiling the parallel encoder scales from. +#[divan::bench(args = [10_000, 100_000], max_time = 5.0)] fn write_events(bencher: Bencher, n: usize) { let events = generate_events(n); + let artifact_bytes = write_frame(&events).len(); + + bencher + .counter(ItemsCount::new(n)) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| write_frame(&events)); +} - bencher.bench_local(|| { - let mut output = Vec::new(); - let mut writer = MemtrackWriter::new(&mut output).unwrap(); - for event in &events { - writer.write_event(event).unwrap(); - } - writer.finish().unwrap(); - }); +fn write_frame(events: &[MemtrackEvent]) -> Vec { + let mut writer = MemtrackWriter::new(Vec::new()).unwrap(); + for event in events { + writer.write_event(event).unwrap(); + } + writer.finish().unwrap() } fn generate_realistic_events(n: usize) -> Vec { @@ -90,12 +122,18 @@ fn generate_realistic_events(n: usize) -> Vec { addr }); let kind = match rng.gen_range(0..20) { - 0 => MemtrackEventKind::Calloc { size }, + 0 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, 1 => MemtrackEventKind::Mmap { size }, - _ => MemtrackEventKind::Malloc { size }, + _ => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, }; - if let MemtrackEventKind::Mmap { size } = kind { - live_mmap.push((addr, size)); + if let MemtrackEventKind::Mmap { size } = &kind { + live_mmap.push((addr, *size)); } else { live_heap.push(addr); } @@ -105,7 +143,7 @@ fn generate_realistic_events(n: usize) -> Vec { if idx < live_heap.len() { let addr = live_heap.swap_remove(idx); free_list.push(addr); - (addr, MemtrackEventKind::Free) + (addr, MemtrackEventKind::Free { stack_hash: 0 }) } else { let (addr, size) = live_mmap.swap_remove(idx - live_heap.len()); free_list.push(addr); @@ -127,6 +165,7 @@ fn generate_realistic_events(n: usize) -> Vec { MemtrackEventKind::Realloc { old_addr: Some(old_addr), size, + stack_hash: 0, }, ) }; @@ -142,15 +181,84 @@ fn generate_realistic_events(n: usize) -> Vec { events } -const REALISTIC_EVENTS: usize = 1_000_000; +/// One event per frame slot of a full window, so every worker count from 1 to +/// `WINDOW_FRAMES` has a frame to take. Sizing below this hides pool scaling: +/// the encoder can only parallelize across whole frames. +const REALISTIC_EVENTS: usize = 16 * 64 * 1024; -#[divan::bench(args = [16, 8, 4], max_time = 10.0)] +/// Throughput of the artifact encoder over a realistic allocation mix, as a +/// function of the worker pool size. +#[divan::bench(args = [1, 2, 4, 8, 16], max_time = 10.0)] fn encode_events_realistic(bencher: Bencher, n_workers: usize) { let events = generate_realistic_events(REALISTIC_EVENTS); + let artifact_bytes = encode(&events, n_workers).len(); - bencher.bench_local(|| { - let mut output = Vec::new(); - encode_events(events.iter().copied(), &mut output, n_workers).unwrap(); - output - }); + bencher + .counter(ItemsCount::new(events.len())) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| encode(&events, n_workers)); +} + +/// Single-frame throughput on captured stacks: each `Stack` event carries a +/// register set and a raw stack copy, so payloads are orders of magnitude +/// larger than an allocation record and the byte rate is what matters. +#[divan::bench(max_time = 10.0)] +fn write_stack_events(bencher: Bencher) { + let events = generate_stack_events(16 * 1024); + let artifact_bytes = write_frame(&events).len(); + + bencher + .counter(ItemsCount::new(events.len())) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| write_frame(&events)); +} + +fn encode(events: &[MemtrackEvent], n_workers: usize) -> Vec { + let mut output = Vec::new(); + encode_events(events.iter().cloned(), &mut output, n_workers).unwrap(); + output +} + +/// A stack-capture heavy stream: one `Stack` record per allocation, sized like +/// the kernel's stack copies (2 KiB payload, x86_64 register set). +fn generate_stack_events(n: usize) -> Vec { + const STACK_BYTES: usize = 2048; + let mut rng = StdRng::seed_from_u64(7); + let mut events = Vec::with_capacity(n * 2); + + while events.len() < n * 2 { + let hash: u64 = rng.r#gen(); + let record = StackRecord { + hash, + sp: 0x7fff_0000_0000 | (rng.gen_range(0..1u64 << 20) << 4), + regs: (0..33).map(|_| rng.r#gen()).collect(), + bytes: (0..STACK_BYTES).map(|_| rng.r#gen()).collect(), + fp_chain: (0..16).map(|_| rng.r#gen()).collect(), + truncated: false, + }; + let addr: u64 = rng.r#gen(); + let timestamp: u64 = rng.r#gen(); + + events.push(MemtrackEvent { + pid: 4242, + tid: 4242, + timestamp, + addr, + kind: MemtrackEventKind::Stack { + record: Box::new(record), + }, + }); + events.push(MemtrackEvent { + pid: 4242, + tid: 4242, + timestamp: timestamp + 1, + addr, + kind: MemtrackEventKind::Malloc { + size: rng.gen_range(8..8192), + stack_hash: hash, + }, + }); + } + + events } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b082a7c67..fed115f29 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -41,7 +41,7 @@ impl MemtrackArtifact { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct MemtrackEvent { pub pid: pid_t, pub tid: pid_t, @@ -51,23 +51,34 @@ pub struct MemtrackEvent { pub kind: MemtrackEventKind, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] pub enum MemtrackEventKind { Malloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, + }, + Free { + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, - Free, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Calloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, AlignedAlloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Mmap { size: u64, @@ -91,6 +102,41 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + /// One executable file mapping from a native PERF_RECORD_MMAP2 record. + /// The common event header carries its address, process, and timestamp. + Mapping { + path: String, + dev: u64, + ino: u64, + file_offset: u64, + len: u64, + }, + + Stack { + // Box keeps the MemtrackEventKind enum small across millions of events. + #[serde(flatten)] + record: Box, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StackRecord { + pub hash: u64, + /// User stack pointer the copy starts at. + pub sp: u64, + /// Registers by DWARF number for the capturing architecture; 33 entries on x86_64. + pub regs: Vec, + /// Raw stack bytes read upward from `sp`. + #[serde(with = "serde_bytes")] + pub bytes: Vec, + /// In-kernel frame-pointer walk, innermost first; empty when unavailable. + pub fp_chain: Vec, + /// The copy filled its budget, so stack above it was not captured. + pub truncated: bool, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 } pub struct MemtrackEventStream { @@ -120,14 +166,17 @@ mod tests { tid: 11, timestamp: 100, addr: 0x10, - kind: MemtrackEventKind::Malloc { size: 64 }, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, }, MemtrackEvent { pid: 1, tid: 12, timestamp: 200, addr: 0x20, - kind: MemtrackEventKind::Free, + kind: MemtrackEventKind::Free { stack_hash: 0 }, }, MemtrackEvent { pid: 1, @@ -139,6 +188,19 @@ mod tests { size: 40960, }, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 400, + addr: 0x400000, + kind: MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, + }, ]; let artifact = MemtrackArtifact { @@ -167,21 +229,54 @@ mod tests { } let kinds = [ - MemtrackEventKind::Malloc { size: 7 }, - MemtrackEventKind::Free, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0, + }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0xCAFE_BABE, + }, + MemtrackEventKind::Free { stack_hash: 0 }, + MemtrackEventKind::Free { stack_hash: 0xFEED }, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, + stack_hash: 0, }, MemtrackEventKind::Realloc { old_addr: None, size: 42, + stack_hash: 0x1234, + }, + MemtrackEventKind::Calloc { + size: 9, + stack_hash: 0, + }, + MemtrackEventKind::AlignedAlloc { + size: 9, + stack_hash: 0, }, - MemtrackEventKind::Calloc { size: 9 }, - MemtrackEventKind::AlignedAlloc { size: 9 }, MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, + MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: 0xDEAD_BEEF, + sp: 0x7FFF_0000, + regs: vec![0; 33], + bytes: vec![1, 2, 3, 4], + fp_chain: vec![0x1000, 0x2000], + truncated: false, + }), + }, ]; for kind in kinds { @@ -190,7 +285,7 @@ mod tests { tid: 42, timestamp: 0xDEAD, addr: 0xBEEF, - kind, + kind: kind.clone(), }; let shadow = Shadow { pid: -7, @@ -215,7 +310,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect(); @@ -265,7 +363,8 @@ mod tests { event.kind, MemtrackEventKind::Realloc { old_addr: None, - size: 42 + size: 42, + stack_hash: 0, } )); diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index c47b3aed9..5cc9a10e2 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -13,6 +13,10 @@ const FRAME_EVENTS: usize = 64 * 1024; /// memory to roughly `FRAME_EVENTS * WINDOW_FRAMES` events regardless of how long /// the source runs. const WINDOW_FRAMES: usize = 16; +/// Compressed bytes to reserve per event, so a frame's output buffer does not +/// grow-and-copy its way up from zero. Overshooting wastes a little memory per +/// in-flight frame; undershooting only costs the doublings it fails to avoid. +const FRAME_BYTES_PER_EVENT: usize = 16; /// Encode a stream of events into a single compressed artifact stream, /// compressing frames in parallel across a Rayon pool of `n_workers` threads. @@ -73,7 +77,7 @@ where /// Encode one batch as a single self-contained zstd frame. fn encode_frame(batch: &[MemtrackEvent]) -> anyhow::Result> { - let mut writer = MemtrackWriter::new(Vec::new())?; + let mut writer = MemtrackWriter::new(Vec::with_capacity(batch.len() * FRAME_BYTES_PER_EVENT))?; for event in batch { writer.write_event(event)?; } @@ -94,7 +98,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect() } diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 7a2c7c89b..7f41a2088 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -1,5 +1,6 @@ use anyhow::Context; use libc::pid_t; +use serde::de::{Deserializer, Error}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::BufWriter; @@ -11,35 +12,68 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; -#[derive(Serialize, Deserialize, Default)] -pub struct WalltimeMetadata { - /// The version of this metadata format. - pub version: u64, - - /// Name and version of the integration - pub integration: (String, String), - - /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub ignored_modules_by_pid: HashMap>, +/// Reads a pid-keyed map whose keys arrive as strings. +/// +/// JSON object keys are always strings. serde_json's direct deserializer +/// special-cases that and parses integer map keys, but a `#[serde(flatten)]` +/// field is buffered into serde's internal `Content` first, and that path has +/// no such special case — a `pid_t` key then fails with `invalid type: string`. +/// See . +/// +/// Only the read side needs this: serializing writes the same bytes either way. +fn pid_keys_from_strings<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} +/// The per-profile module artifacts: the deduplicated debug info, unwind data +/// and symbol tables extracted from the ELF modules the profiled processes +/// mapped, plus the per-pid references into them. +/// +/// Flattened into every metadata format, so all profiling modes describe their +/// modules identically. +#[derive(Serialize, Deserialize, Default)] +pub struct ModuleArtifacts { /// Deduplicated debug info entries, keyed by semantic key #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub debug_info: HashMap, /// Per-pid debug info references, mapping PID to mounted modules' debug info /// Referenced by `path_keys` that point to the deduplicated `debug_info` entries. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + deserialize_with = "pid_keys_from_strings" + )] pub mapped_process_debug_info_by_pid: HashMap>, /// Per-pid unwind data references, mapping PID to mounted modules' unwind data /// Referenced by `path_keys` that point to the deduplicated `unwind_data` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + deserialize_with = "pid_keys_from_strings" + )] pub mapped_process_unwind_data_by_pid: HashMap>, /// Per-pid symbol references, mapping PID to its mounted modules' symbols /// Referenced by `path_keys` that point to the deduplicated `symbols.map` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + deserialize_with = "pid_keys_from_strings" + )] pub mapped_process_module_symbols: HashMap>, /// Mapping from semantic `path_key` to original binary path on host disk @@ -49,6 +83,22 @@ pub struct WalltimeMetadata { /// Until now, only kept for traceability, if we ever need to reconstruct the original paths from the keys #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub path_key_to_path: HashMap, +} + +#[derive(Serialize, Deserialize, Default)] +pub struct WalltimeMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub ignored_modules_by_pid: HashMap>, + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, // Deprecated fields below are kept for backward compatibility, since this struct is used in // the parser and older versions of the runner still generate them @@ -85,3 +135,152 @@ impl WalltimeMetadata { Ok(()) } } + +/// Companion to the memtrack event stream: the modules its allocation stacks +/// resolve against. Memory mode records benchmark boundaries in +/// `ExecutionTimestamps`, so unlike [`WalltimeMetadata`] it carries no markers. +#[derive(Serialize, Deserialize, Default)] +pub struct MemtrackMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, +} + +impl MemtrackMetadata { + pub const CURRENT_VERSION: u64 = 1; + + pub fn new(integration: (String, String), artifacts: ModuleArtifacts) -> Self { + Self { + version: Self::CURRENT_VERSION, + integration, + artifacts, + } + } + + pub fn from_reader(reader: R) -> anyhow::Result { + serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") + } + + pub fn save_to>(&self, path: P) -> anyhow::Result<()> { + let file = std::fs::File::create(path.as_ref().join("memtrack.metadata"))?; + const BUFFER_SIZE: usize = 256 * 1024 /* 256 KB */; + + let writer = BufWriter::with_capacity(BUFFER_SIZE, file); + serde_json::to_writer(writer, self)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Captured from the flat `WalltimeMetadata` that predates + /// [`ModuleArtifacts`]: flattening must not move a single byte, since the + /// parser reads this format from runners of every version. + const WALLTIME_JSON: &str = r#"{"version":7,"integration":["codspeed-rust","4.2.0"],"ignored_modules_by_pid":{"42":[["/lib/libpython.so",4096,8192]]},"debug_info":{"0__libc.so.6":{"object_path":"/lib/libc.so.6","addr_bounds":[4096,36864],"load_bias":4096,"debug_infos":[{"addr":4352,"size":32,"name":"malloc","file":"malloc.c","line":11}]}},"mapped_process_debug_info_by_pid":{"42":[{"debug_info_key":"0__libc.so.6","load_bias":4096}]},"mapped_process_unwind_data_by_pid":{"42":[{"unwind_data_key":"0__libc.so.6","timestamp":1234,"avma_range":{"start":4096,"end":36864},"base_avma":4096}]},"mapped_process_module_symbols":{"42":[{"perf_map_key":"0__libc.so.6","load_bias":4096}]},"path_key_to_path":{"0__libc.so.6":"/lib/libc.so.6"},"uri_by_ts":[[1,"bench::a"]],"ignored_modules":[],"markers":[]}"#; + + fn populated_artifacts() -> ModuleArtifacts { + ModuleArtifacts { + debug_info: HashMap::from([( + "0__libc.so.6".to_string(), + ModuleDebugInfo { + object_path: "/lib/libc.so.6".to_string(), + addr_bounds: (0x1000, 0x9000), + load_bias: 0x1000, + debug_infos: vec![crate::debug_info::DebugInfo { + addr: 0x1100, + size: 0x20, + name: "malloc".to_string(), + file: "malloc.c".to_string(), + line: Some(11), + }], + }, + )]), + mapped_process_debug_info_by_pid: HashMap::from([( + 42, + vec![MappedProcessDebugInfo { + debug_info_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + mapped_process_unwind_data_by_pid: HashMap::from([( + 42, + vec![MappedProcessUnwindData { + unwind_data_key: "0__libc.so.6".to_string(), + inner: crate::unwind_data::ProcessUnwindData { + timestamp: Some(1234), + avma_range: 0x1000..0x9000, + base_avma: 0x1000, + }, + }], + )]), + mapped_process_module_symbols: HashMap::from([( + 42, + vec![crate::module_symbols::MappedProcessModuleSymbols { + perf_map_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + path_key_to_path: HashMap::from([( + "0__libc.so.6".to_string(), + PathBuf::from("/lib/libc.so.6"), + )]), + } + } + + #[test] + fn walltime_metadata_serialization_is_unchanged_by_flattening() { + #[allow(deprecated)] + let metadata = WalltimeMetadata { + version: 7, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + ignored_modules_by_pid: HashMap::from([( + 42, + vec![("/lib/libpython.so".to_string(), 0x1000, 0x2000)], + )]), + artifacts: populated_artifacts(), + uri_by_ts: vec![(1, "bench::a".to_string())], + ignored_modules: vec![], + markers: vec![], + debug_info_by_pid: HashMap::new(), + }; + + assert_eq!(serde_json::to_string(&metadata).unwrap(), WALLTIME_JSON); + } + + #[test] + fn walltime_metadata_round_trips_through_the_flattened_fields() { + let parsed = WalltimeMetadata::from_reader(WALLTIME_JSON.as_bytes()).unwrap(); + + assert_eq!(parsed.artifacts.path_key_to_path.len(), 1); + assert_eq!( + parsed.artifacts.mapped_process_unwind_data_by_pid[&42].len(), + 1 + ); + assert_eq!(serde_json::to_string(&parsed).unwrap(), WALLTIME_JSON); + } + + #[test] + fn memtrack_metadata_round_trips() { + let metadata = MemtrackMetadata { + version: MemtrackMetadata::CURRENT_VERSION, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + artifacts: populated_artifacts(), + }; + + let json = serde_json::to_string(&metadata).unwrap(); + let parsed = MemtrackMetadata::from_reader(json.as_bytes()).unwrap(); + + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + assert_eq!( + parsed.artifacts.mapped_process_module_symbols[&42][0].perf_map_key, + "0__libc.so.6" + ); + } +} diff --git a/src/cli/exec/mod.rs b/src/cli/exec/mod.rs index 4a757f74b..e7ed8b5e2 100644 --- a/src/cli/exec/mod.rs +++ b/src/cli/exec/mod.rs @@ -93,6 +93,7 @@ fn build_orchestrator_config( cycle_estimation: args.shared.cycle_estimation, exclude_allocations: args.shared.exclude_allocations, simulation_track_subprocess: args.shared.simulation_track_subprocess, + memory_capture_stack: args.shared.experimental.experimental_memory_capture_stack, }) } diff --git a/src/cli/experimental.rs b/src/cli/experimental.rs index 74aab8177..196967568 100644 --- a/src/cli/experimental.rs +++ b/src/cli/experimental.rs @@ -17,6 +17,15 @@ pub struct ExperimentalArgs { )] pub experimental_fair_sched: bool, + /// Capture allocation call stacks in memory mode. + #[arg( + long, + default_value_t = false, + help_heading = "Experimental", + env = "CODSPEED_EXPERIMENTAL_MEMORY_CAPTURE_STACK" + )] + pub experimental_memory_capture_stack: bool, + /// Deprecated: cycle estimation is enabled by default and this flag has no effect. #[arg(long, hide = true, env = "CODSPEED_EXPERIMENTAL_CYCLE_ESTIMATION")] pub experimental_cycle_estimation: bool, @@ -33,6 +42,9 @@ impl ExperimentalArgs { if self.experimental_fair_sched { flags.push("--experimental-fair-sched"); } + if self.experimental_memory_capture_stack { + flags.push("--experimental-memory-capture-stack"); + } flags } diff --git a/src/cli/run/mod.rs b/src/cli/run/mod.rs index a218155c7..86bc8440a 100644 --- a/src/cli/run/mod.rs +++ b/src/cli/run/mod.rs @@ -81,6 +81,7 @@ impl RunArgs { }, experimental: ExperimentalArgs { experimental_fair_sched: false, + experimental_memory_capture_stack: false, experimental_cycle_estimation: false, experimental_exclude_allocations: false, }, @@ -135,6 +136,7 @@ fn build_orchestrator_config( cycle_estimation: args.shared.cycle_estimation, exclude_allocations: args.shared.exclude_allocations, simulation_track_subprocess: args.shared.simulation_track_subprocess, + memory_capture_stack: args.shared.experimental.experimental_memory_capture_stack, }) } diff --git a/src/executor/config.rs b/src/executor/config.rs index 07f99c820..e39e559e3 100644 --- a/src/executor/config.rs +++ b/src/executor/config.rs @@ -98,6 +98,9 @@ pub struct OrchestratorConfig { /// Inherit valgrind's instrumentation state across a traced exec, so the cost of /// subprocesses spawned by a benchmark is measured too. pub simulation_track_subprocess: bool, + /// Capture allocation call stacks in memory mode, so allocations can be + /// attributed to the code that made them. + pub memory_capture_stack: bool, } /// Per-execution configuration passed to executors. @@ -138,6 +141,9 @@ pub struct ExecutorConfig { /// Inherit valgrind's instrumentation state across a traced exec, so the cost of /// subprocesses spawned by a benchmark is measured too. pub simulation_track_subprocess: bool, + /// Capture allocation call stacks in memory mode, so allocations can be + /// attributed to the code that made them. + pub memory_capture_stack: bool, } #[derive(Debug, Clone, PartialEq)] @@ -210,6 +216,7 @@ impl OrchestratorConfig { cycle_estimation: self.cycle_estimation, exclude_allocations: self.exclude_allocations, simulation_track_subprocess: self.simulation_track_subprocess, + memory_capture_stack: self.memory_capture_stack, } } } @@ -245,6 +252,7 @@ impl OrchestratorConfig { cycle_estimation: true, exclude_allocations: false, simulation_track_subprocess: false, + memory_capture_stack: false, } } } diff --git a/src/executor/helpers/debug_file.rs b/src/executor/helpers/debug_file.rs index 0619bed2b..bb1c9d2ba 100644 --- a/src/executor/helpers/debug_file.rs +++ b/src/executor/helpers/debug_file.rs @@ -12,11 +12,17 @@ use std::path::{Path, PathBuf}; /// /// [Separate Debug Files]: https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html pub fn find_debug_file(object: &object::File, binary_path: &Path) -> Option { - ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] + let via_known_roots = ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] .iter() .map(Path::new) .filter(|dir| dir.exists()) - .find_map(|dir| find_debug_file_in(object, binary_path, dir)) + .find_map(|dir| find_debug_file_in(object, binary_path, dir)); + + // `.gnu_debuglink` beside the binary (and its `.debug` subdirectory) is part of + // GDB's search order regardless of whether a global debug-directory root exists + // (see module docs), so it must not be skipped on systems without one, e.g. most + // CI runners. + via_known_roots.or_else(|| find_debug_file_by_debuglink(object, binary_path, None)) } fn find_debug_file_in( @@ -27,7 +33,7 @@ fn find_debug_file_in( if let Some(path) = find_debug_file_by_build_id(object, debug_dir) { return Some(path); } - find_debug_file_by_debuglink(object, binary_path, debug_dir) + find_debug_file_by_debuglink(object, binary_path, Some(debug_dir)) } /// Build-id `a05cfb6313fe06a13c9b4b5cb86c2069faa3951f` resolves to @@ -58,19 +64,20 @@ fn find_debug_file_by_build_id(object: &object::File, debug_dir: &Path) -> Optio fn find_debug_file_by_debuglink( object: &object::File, binary_path: &Path, - debug_dir: &Path, + debug_dir: Option<&Path>, ) -> Option { let (debuglink, expected_crc) = object.gnu_debuglink().ok()??; let debuglink = std::str::from_utf8(debuglink).ok()?; let dir = binary_path.parent()?; - let candidates = [ - dir.join(debuglink), - dir.join(".debug").join(debuglink), - debug_dir - .join(dir.strip_prefix("/").unwrap_or(dir)) - .join(debuglink), - ]; + let mut candidates = vec![dir.join(debuglink), dir.join(".debug").join(debuglink)]; + if let Some(debug_dir) = debug_dir { + candidates.push( + debug_dir + .join(dir.strip_prefix("/").unwrap_or(dir)) + .join(debuglink), + ); + } candidates.into_iter().find(|p| { let Ok(content) = std::fs::read(p) else { diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8c9a3985..6e53a3e9b 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback; use crate::executor::helpers::run_with_env::prefix_command_with_env; use crate::executor::helpers::run_with_sudo::is_root_user; +use crate::executor::memory::module_artifacts::save_module_artifacts; use crate::executor::memory::tunables::MemoryTunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; @@ -24,6 +25,7 @@ use runner_shared::artifacts::{ArtifactExt, ExecutionTimestamps}; use runner_shared::fifo::Command as FifoCommand; use runner_shared::fifo::IntegrationMode; use semver::Version; +use std::cell::RefCell; use std::fs::canonicalize; use std::path::Path; use std::rc::Rc; @@ -69,6 +71,9 @@ impl MemoryExecutor { cmd_builder.arg("--ipc-server"); cmd_builder.arg(server_name); cmd_builder.arg(bench_command); + if execution_context.config.memory_capture_stack { + cmd_builder.env("CODSPEED_MEMTRACK_CAPTURE_STACKS", "1"); + } // Set working directory if specified if let Some(cwd) = &execution_context.config.working_directory { @@ -163,7 +168,8 @@ impl Executor for MemoryExecutor { let _tunables = MemoryTunables::apply(); // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions - std::fs::create_dir_all(execution_context.profile_folder.join("results"))?; + let results_folder = execution_context.profile_folder.join("results"); + std::fs::create_dir_all(&results_folder)?; Self::ensure_privileges()?; @@ -172,16 +178,18 @@ impl Executor for MemoryExecutor { debug!("cmd: {cmd:?}"); let runner_fifo = RunnerFifo::new()?; - let on_process_started = |mut child: std::process::Child| async move { - let (marker_result, exit_status) = - Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + let integration = Rc::new(RefCell::new(None)); + let on_process_started = { + let integration = integration.clone(); + |mut child: std::process::Child| async move { + let (marker_result, fifo_data, exit_status) = + Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + *integration.borrow_mut() = fifo_data.integration; - // Directly write to the profile folder, to avoid having to define another field - marker_result - .save_to(execution_context.profile_folder.join("results")) - .unwrap(); + marker_result.save_to(&results_folder).unwrap(); - Ok(exit_status) + Ok(exit_status) + } }; let status = run_command_with_log_pipe_and_callback(cmd, on_process_started).await?; @@ -191,6 +199,19 @@ impl Executor for MemoryExecutor { bail!("failed to execute memory tracker process: {status}"); } + if let Some(integration) = integration.borrow_mut().take() { + let results_folder = execution_context.profile_folder.join("results"); + if let Err(e) = save_module_artifacts( + &execution_context.profile_folder, + &results_folder, + integration, + ) { + // The memory results are complete without them; only offline + // stack attribution is lost. + error!("Failed to save memtrack module artifacts: {e:#}"); + } + } + Ok(()) } @@ -228,7 +249,11 @@ impl MemoryExecutor { mut runner_fifo: RunnerFifo, ipc: MemtrackIpcServer, child: &mut std::process::Child, - ) -> anyhow::Result<(ExecutionTimestamps, std::process::ExitStatus)> { + ) -> anyhow::Result<( + ExecutionTimestamps, + crate::executor::shared::fifo::FifoBenchmarkData, + std::process::ExitStatus, + )> { // Accept the IPC connection from memtrack and get the sender it sends us // Use a timeout to prevent hanging if the process doesn't start properly // https://github.com/servo/ipc-channel/issues/261 @@ -300,9 +325,9 @@ impl MemoryExecutor { Ok(None) }; - let (marker_result, _, exit_status) = + let (marker_result, fifo_data, exit_status) = runner_fifo.handle_fifo_messages(child, on_cmd).await?; - Ok((marker_result, exit_status)) + Ok((marker_result, fifo_data, exit_status)) } } diff --git a/src/executor/memory/mod.rs b/src/executor/memory/mod.rs index 2d17547d1..9f48a81ab 100644 --- a/src/executor/memory/mod.rs +++ b/src/executor/memory/mod.rs @@ -1,3 +1,4 @@ pub mod executor; +pub(crate) mod module_artifacts; pub(crate) mod setup; pub(crate) mod tunables; diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs new file mode 100644 index 000000000..528619d5f --- /dev/null +++ b/src/executor/memory/module_artifacts.rs @@ -0,0 +1,669 @@ +use crate::executor::shared::module_artifacts::loaded_module::LoadedModule; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; +use crate::prelude::*; +use libc::pid_t; +use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind}; +use runner_shared::metadata::MemtrackMetadata; +use runner_shared::unwind_data::ProcessUnwindData; +use std::collections::HashMap; +use std::ops::Range; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProcessMapping { + pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + dev: u64, + ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + file_offset: u64, + avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + timestamp: u64, +} + +/// Turn the mappings memtrack recorded into the artifacts an offline unwinder +/// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the +/// `memtrack.metadata` referencing them per pid. +/// +/// `results_folder` is where memtrack wrote its artifacts; the keyed files and +/// the metadata land in `profile_folder`, next to walltime's equivalents. +pub fn save_module_artifacts( + profile_folder: &Path, + results_folder: &Path, + integration: (String, String), +) -> Result<()> { + let mappings = read_mappings(results_folder)?; + if mappings.is_empty() { + debug!("No module mappings recorded, skipping memtrack module artifacts"); + return Ok(()); + } + + let loaded_modules = loaded_modules_from_mappings(&mappings); + debug!( + "Extracting artifacts for {} modules from {} mappings", + loaded_modules.len(), + mappings.len() + ); + + let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); + MemtrackMetadata::new(integration, saved.artifacts).save_to(profile_folder) +} + +fn read_mappings(results_folder: &Path) -> Result> { + let suffix = format!(".{}.msgpack", MemtrackArtifact::name()); + + let mut mappings = Vec::new(); + for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { + if !entry.file_name().to_string_lossy().ends_with(&suffix) { + continue; + } + + let file = std::fs::File::open(entry.path())?; + mappings.extend( + read_mappings_from_artifact(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?, + ); + } + + mappings.sort_unstable_by_key(|mapping| (mapping.pid, mapping.timestamp)); + Ok(mappings) +} + +/// Reconstruct mappings across forks because inherited perf events do not +/// synthesize mappings that already existed when a child was forked. +fn read_mappings_from_artifact(reader: R) -> Result> { + let mut timeline = MemtrackArtifact::decode_streamed(reader)? + .filter(|event| { + matches!( + &event.kind, + MemtrackEventKind::Exec + | MemtrackEventKind::Mapping { .. } + | MemtrackEventKind::Fork { .. } + ) + }) + .collect::>(); + + // Ties break so exec purges before mapping, while fork inherits that mapping. + timeline.sort_by_key(|event| { + let rank = match &event.kind { + MemtrackEventKind::Exec => 0, + MemtrackEventKind::Mapping { .. } => 1, + MemtrackEventKind::Fork { .. } => 2, + _ => unreachable!(), + }; + (event.timestamp, rank) + }); + + let mut live_mappings: HashMap> = HashMap::new(); + let mut mappings = Vec::new(); + + for event in timeline { + match event.kind { + MemtrackEventKind::Mapping { + path, + dev, + ino, + file_offset, + len, + } => { + let Some(end) = event.addr.checked_add(len) else { + debug!("Skipping mapping for {path}: address range overflows"); + continue; + }; + + let mapping = ProcessMapping { + pid: event.pid, + path, + dev, + ino, + file_offset, + avma_range: event.addr..end, + timestamp: event.timestamp, + }; + live_mappings + .entry(mapping.pid) + .or_default() + .push(mapping.clone()); + mappings.push(mapping); + } + MemtrackEventKind::Fork { parent_pid } => { + let inherited = live_mappings.get(&parent_pid).cloned().unwrap_or_default(); + let child_mappings = inherited + .into_iter() + .map(|mut mapping| { + mapping.pid = event.pid; + mapping.timestamp = event.timestamp; + mapping + }) + .collect::>(); + mappings.extend(child_mappings.iter().cloned()); + live_mappings.insert(event.pid, child_mappings); + } + MemtrackEventKind::Exec => { + live_mappings.remove(&event.pid); + } + _ => unreachable!(), + } + } + + Ok(mappings) +} + +fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { + let mut loaded_modules = HashMap::::new(); + + for mapping in mappings { + let path = PathBuf::from(&mapping.path); + if !names_mapped_file(mapping, &path) { + continue; + } + + let load_bias = match ModuleSymbols::compute_load_bias( + &path, + mapping.avma_range.start, + mapping.avma_range.end, + mapping.file_offset, + ) { + Ok(load_bias) => load_bias, + Err(e) => { + debug!("Failed to compute load bias for {}: {e}", mapping.path); + continue; + } + }; + + let loaded_module = loaded_modules.entry(path.clone()).or_default(); + + if loaded_module.module_symbols.is_none() { + match ModuleSymbols::from_elf(&path) { + Ok(symbols) => loaded_module.module_symbols = Some(symbols), + Err(e) => debug!("Failed to load symbols for {}: {e}", mapping.path), + } + } + + let process_unwind_data = if let Some(unwind_data) = &loaded_module.unwind_data { + Some(ProcessUnwindData { + timestamp: Some(mapping.timestamp), + avma_range: mapping.avma_range.clone(), + base_avma: unwind_data.base_svma.wrapping_add(load_bias), + }) + } else { + match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + loaded_module.unwind_data = Some(unwind_data); + process_unwind_data.timestamp = Some(mapping.timestamp); + Some(process_unwind_data) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } + } + }; + + let process_loaded_module = loaded_module + .process_loaded_modules + .entry(mapping.pid) + .or_default(); + process_loaded_module.symbols_load_bias = Some(load_bias); + + if let Some(process_unwind_data) = process_unwind_data { + process_loaded_module.process_unwind_data = Some(process_unwind_data); + } + } + + loaded_modules +} + +/// Whether the path still names the file that was mapped. +/// +/// The mapping records the inode the kernel resolved the path from; a file +/// rebuilt or replaced since then is a different inode, and reading unwind data +/// out of it would bind eh_frame from the wrong binary to those addresses. +fn names_mapped_file(mapping: &ProcessMapping, path: &Path) -> bool { + let Ok(metadata) = std::fs::metadata(path) else { + debug!("{} is no longer readable", mapping.path); + return false; + }; + + // The recorded `dev` is the kernel's s_dev encoding, `st_dev` glibc's, so + // only the decomposed major/minor pair is comparable. + let recorded = (mapping.dev >> 20, mapping.dev & 0xF_FFFF, mapping.ino); + let current = ( + u64::from(libc::major(metadata.dev())), + u64::from(libc::minor(metadata.dev())), + metadata.ino(), + ); + + if recorded != current { + debug!( + "{} changed since it was mapped (recorded {recorded:?}, now {current:?})", + mapping.path + ); + return false; + } + true +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + use runner_shared::artifacts::MemtrackEvent; + + fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { + ProcessMapping { + pid: 42, + path: path.to_string(), + dev, + ino, + file_offset: 0, + avma_range: 0x1000..0x2000, + timestamp: 7, + } + } + + fn s_dev_of(path: &str) -> (u64, u64) { + let metadata = std::fs::metadata(path).unwrap(); + let dev = + u64::from(libc::major(metadata.dev())) << 20 | u64::from(libc::minor(metadata.dev())); + (dev, metadata.ino()) + } + + /// The recorded s_dev encoding and `st_dev` differ, so the check has to + /// decompose both or it rejects every module that did not change. + #[test] + fn accepts_a_file_that_still_has_the_recorded_inode() { + let path = "/proc/self/exe"; + let (dev, ino) = s_dev_of(path); + + assert!(names_mapped_file( + &mapping_for(path, dev, ino), + Path::new(path) + )); + } + + #[test] + fn rejects_a_file_whose_inode_changed() { + let path = "/proc/self/exe"; + let (dev, _) = s_dev_of(path); + + assert!(!names_mapped_file( + &mapping_for(path, dev, 0), + Path::new(path) + )); + } + + #[test] + fn rejects_a_path_that_no_longer_exists() { + let path = "/nonexistent/module.so"; + + assert!(!names_mapped_file( + &mapping_for(path, 1, 2), + Path::new(path) + )); + } + + #[test] + fn sorts_interleaved_mapping_artifacts_by_pid_and_timestamp() { + let results = tempfile::tempdir().unwrap(); + + // Separate files model records drained from different per-CPU rings. + MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 20, + addr: 0x2000, + kind: MemtrackEventKind::Mapping { + path: "second-module.so".to_string(), + dev: 2, + ino: 2, + file_offset: 0x2000, + len: 0x1000, + }, + }, + MemtrackEvent { + pid: 7, + tid: 7, + timestamp: 30, + addr: 0x7000, + kind: MemtrackEventKind::Mapping { + path: "child-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0x3000, + len: 0x1000, + }, + }, + ], + } + .save_file_to(results.path(), "cpu1.MemtrackArtifact.msgpack") + .unwrap(); + MemtrackArtifact { + events: vec![MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 10, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: "first-module.so".to_string(), + dev: 1, + ino: 1, + file_offset: 0x1000, + len: 0x1000, + }, + }], + } + .save_file_to(results.path(), "cpu0.MemtrackArtifact.msgpack") + .unwrap(); + + let mappings = read_mappings(results.path()).unwrap(); + assert_eq!( + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.timestamp)) + .collect::>(), + vec![(7, 30), (42, 10), (42, 20)] + ); + assert_eq!(mappings[0].path, "child-module.so"); + assert_eq!(mappings[1].path, "first-module.so"); + assert_eq!(mappings[2].path, "second-module.so"); + } + #[test] + fn extracts_all_mapping_events_from_a_streamed_artifact() { + const FIRST_MODULE: &str = "first-module.so"; + const SECOND_MODULE: &str = "second-module.so"; + let artifact = MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 10, + addr: 0, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, + }, + MemtrackEvent { + pid: 7, + tid: 8, + timestamp: 11, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + len: 0x2000, + }, + }, + MemtrackEvent { + pid: 9, + tid: 10, + timestamp: 12, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + len: 0x3000, + }, + }, + MemtrackEvent { + pid: 99, + tid: 99, + timestamp: 99, + addr: u64::MAX, + kind: MemtrackEventKind::Mapping { + path: "overflow-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0, + len: 1, + }, + }, + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 13, + addr: 0x2000, + kind: MemtrackEventKind::Free { stack_hash: 0 }, + }, + ], + }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + + assert_eq!( + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap(), + vec![ + ProcessMapping { + pid: 7, + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + avma_range: 0x1000..0x3000, + timestamp: 11, + }, + ProcessMapping { + pid: 9, + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + avma_range: 0x4000..0x7000, + timestamp: 12, + }, + ] + ); + } + + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_streamed_mapping() { + const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; + + let profile = tempfile::tempdir().unwrap(); + let results = profile.path().join("results"); + std::fs::create_dir_all(&results).unwrap(); + + let (dev, ino) = s_dev_of(MODULE); + MemtrackArtifact { + events: vec![MemtrackEvent { + pid: 1234, + tid: 1234, + timestamp: 999, + addr: 0x5555_555a_7000, + kind: MemtrackEventKind::Mapping { + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + len: 0x109_000, + }, + }], + } + .save_with_pid_to(&results, 1234) + .unwrap(); + + save_module_artifacts( + profile.path(), + &results, + ("codspeed-rust".to_string(), "4.2.0".to_string()), + ) + .unwrap(); + + let metadata = MemtrackMetadata::from_reader( + std::fs::File::open(profile.path().join("memtrack.metadata")).unwrap(), + ) + .unwrap(); + + assert_eq!(metadata.version, MemtrackMetadata::CURRENT_VERSION); + assert_eq!( + metadata.artifacts.mapped_process_module_symbols[&1234].len(), + 1 + ); + + let unwind = &metadata.artifacts.mapped_process_unwind_data_by_pid[&1234][0]; + assert_eq!(unwind.inner.timestamp, Some(999)); + assert!( + profile + .path() + .join(format!("{}.unwind_data", unwind.unwind_data_key)) + .exists() + ); + assert_eq!( + metadata.artifacts.path_key_to_path[&unwind.unwind_data_key], + PathBuf::from(MODULE) + ); + } + + fn mapping_event(pid: pid_t, timestamp: u64, addr: u64, path: &str) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr, + kind: MemtrackEventKind::Mapping { + path: path.to_string(), + dev: 1, + ino: 1, + file_offset: 0, + len: 0x1000, + }, + } + } + + fn fork_event(child_pid: pid_t, parent_pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid: child_pid, + tid: child_pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Fork { parent_pid }, + } + } + + fn exec_event(pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Exec, + } + } + + fn decode_lifecycle(events: Vec) -> Vec { + let artifact = MemtrackArtifact { events }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap() + } + + fn mapping_summary(mappings: &[ProcessMapping]) -> Vec<(pid_t, &str, u64)> { + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.path.as_str(), mapping.timestamp)) + .collect() + } + + #[test] + fn fork_without_exec_inherits_only_mappings_before_fork() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + mapping_event(100, 30, 0x2000, "after-fork.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 30), + ] + ); + } + + #[test] + fn exec_stops_inheriting_parent_mappings() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + exec_event(200, 30), + mapping_event(100, 40, 0x2000, "after-fork.so"), + mapping_event(200, 50, 0x3000, "after-exec.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 40), + (200, "after-exec.so", 50), + ] + ); + } + + #[test] + fn grandchild_inherits_transitively_from_forked_child() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "root.so"), + fork_event(200, 100, 20), + mapping_event(200, 25, 0x2000, "child.so"), + fork_event(300, 200, 30), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "root.so", 10), + (200, "root.so", 20), + (200, "child.so", 25), + (300, "root.so", 30), + (300, "child.so", 30), + ] + ); + } + + #[test] + fn equal_timestamp_events_follow_exec_mapping_fork_rank() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-exec.so"), + fork_event(200, 100, 20), + mapping_event(100, 20, 0x2000, "after-exec.so"), + exec_event(100, 20), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-exec.so", 10), + (100, "after-exec.so", 20), + (200, "after-exec.so", 20), + ] + ); + } +} diff --git a/src/executor/shared/mod.rs b/src/executor/shared/mod.rs index 2badf4064..f278f07cd 100644 --- a/src/executor/shared/mod.rs +++ b/src/executor/shared/mod.rs @@ -1 +1,2 @@ pub mod fifo; +pub mod module_artifacts; diff --git a/src/executor/wall_time/profiler/perf/debug_info.rs b/src/executor/shared/module_artifacts/debug_info.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/debug_info.rs rename to src/executor/shared/module_artifacts/debug_info.rs diff --git a/src/executor/wall_time/profiler/perf/elf_helper.rs b/src/executor/shared/module_artifacts/elf_helper.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/elf_helper.rs rename to src/executor/shared/module_artifacts/elf_helper.rs diff --git a/src/executor/wall_time/profiler/perf/loaded_module.rs b/src/executor/shared/module_artifacts/loaded_module.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/loaded_module.rs rename to src/executor/shared/module_artifacts/loaded_module.rs diff --git a/src/executor/shared/module_artifacts/mod.rs b/src/executor/shared/module_artifacts/mod.rs new file mode 100644 index 000000000..34528e0ad --- /dev/null +++ b/src/executor/shared/module_artifacts/mod.rs @@ -0,0 +1,13 @@ +//! Extract symbols, unwind data, and debug info from mapped ELF modules. +//! +//! The input is a set of [`loaded_module::LoadedModule`] values. The output is +//! keyed `unwind_data`/`symbols.map` files and per-process metadata references. + +mod elf_helper; +mod naming; + +pub mod debug_info; +pub mod loaded_module; +pub mod module_symbols; +pub mod save_artifacts; +pub mod unwind_data; diff --git a/src/executor/wall_time/profiler/perf/module_symbols.rs b/src/executor/shared/module_artifacts/module_symbols.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/module_symbols.rs rename to src/executor/shared/module_artifacts/module_symbols.rs diff --git a/src/executor/wall_time/profiler/perf/naming.rs b/src/executor/shared/module_artifacts/naming.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/naming.rs rename to src/executor/shared/module_artifacts/naming.rs diff --git a/src/executor/wall_time/profiler/perf/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs similarity index 93% rename from src/executor/wall_time/profiler/perf/save_artifacts.rs rename to src/executor/shared/module_artifacts/save_artifacts.rs index 36b2fd12a..3e8903ed9 100644 --- a/src/executor/wall_time/profiler/perf/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -1,23 +1,22 @@ use super::debug_info::debug_info_by_path; use super::loaded_module::LoadedModule; +use super::naming; use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; -use crate::executor::wall_time::profiler::perf::naming; use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; use runner_shared::debug_info::{MappedProcessDebugInfo, ModuleDebugInfo}; +use runner_shared::metadata::ModuleArtifacts; use runner_shared::module_symbols::MappedProcessModuleSymbols; use runner_shared::unwind_data::{MappedProcessUnwindData, ProcessUnwindData, UnwindData}; use std::collections::HashMap; use std::path::{Path, PathBuf}; pub struct SavedArtifacts { - pub symbol_pid_mappings_by_pid: HashMap>, - pub debug_info: HashMap, - pub mapped_process_debug_info_by_pid: HashMap>, - pub mapped_process_unwind_data_by_pid: HashMap>, + pub artifacts: ModuleArtifacts, + /// Kept out of [`ModuleArtifacts`] because only the folded walltime trace + /// drops modules; other modes carry every module they mapped. pub ignored_modules_by_pid: HashMap>, - pub key_to_path: HashMap, } /// Save all artifacts (symbols, debug info, unwind data) from mounted modules and JIT data. @@ -30,7 +29,7 @@ pub fn save_artifacts( register_paths(&mut path_to_key, loaded_modules_by_path); - let symbol_pid_mappings_by_pid = + let mapped_process_module_symbols = save_symbols(profile_folder, loaded_modules_by_path, &path_to_key); let (debug_info, mapped_process_debug_info_by_pid) = @@ -45,18 +44,20 @@ pub fn save_artifacts( let ignored_modules_by_pid = collect_ignored_modules(loaded_modules_by_path); - let key_to_path = path_to_key + let path_key_to_path = path_to_key .into_iter() .map(|(path, key)| (key, path)) .collect(); SavedArtifacts { - symbol_pid_mappings_by_pid, - debug_info, - mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid, + artifacts: ModuleArtifacts { + debug_info, + mapped_process_debug_info_by_pid, + mapped_process_unwind_data_by_pid, + mapped_process_module_symbols, + path_key_to_path, + }, ignored_modules_by_pid, - key_to_path, } } diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap index 48d654070..9b917e545 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap index e92dcefa8..5b6a04ae2 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap index 75d0a4494..97dc7b43a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap index 6cf90c6a1..fd5e1aa0c 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap index 9e9c52a2e..a238cbb28 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap index 8456dd05b..34c79e04e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap index 84138e7bb..990e660d9 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap index 879d29f90..fe5907dd0 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap index 839039f35..10a77b716 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap index fec3e2802..724f3002e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap index 205b5e148..6c554024a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap index 699e4b031..807e10611 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap index a0a5b0f98..3956fd7d1 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap index 0367c2dee..edfd8e558 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap index 9fc15dca2..066c1ad0e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/unwind_data.rs b/src/executor/shared/module_artifacts/unwind_data.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/unwind_data.rs rename to src/executor/shared/module_artifacts/unwind_data.rs diff --git a/src/executor/wall_time/profiler/perf/jit_dump.rs b/src/executor/wall_time/profiler/perf/jit_dump.rs index fd5fad056..344f4080e 100644 --- a/src/executor/wall_time/profiler/perf/jit_dump.rs +++ b/src/executor/wall_time/profiler/perf/jit_dump.rs @@ -1,4 +1,4 @@ -use super::module_symbols::{ModuleSymbols, Symbol}; +use crate::executor::shared::module_artifacts::module_symbols::{ModuleSymbols, Symbol}; use crate::prelude::*; use linux_perf_data::jitdump::{JitDumpReader, JitDumpRecord}; use runner_shared::unwind_data::{ProcessUnwindData, UnwindData}; diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 8816e5fb0..7f31921e2 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -10,6 +10,7 @@ use crate::executor::helpers::env::suppress_go_perf_unwinding_warning; use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; +use crate::executor::shared::module_artifacts::save_artifacts; use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING; use crate::executor::wall_time::profiler::Profiler; use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ; @@ -29,16 +30,9 @@ use runner_shared::metadata::WalltimeMetadata; use std::path::Path; use std::path::PathBuf; -mod debug_info; -mod elf_helper; mod jit_dump; -mod loaded_module; -mod module_symbols; -mod naming; mod parse_perf_file; -mod save_artifacts; pub(crate) mod setup; -mod unwind_data; pub mod fifo; pub mod perf_executable; @@ -306,11 +300,7 @@ impl BenchmarkData<'_> { uri_by_ts: self.marker_result.uri_by_ts.clone(), ignored_modules_by_pid: artifacts.ignored_modules_by_pid, markers: self.marker_result.markers.clone(), - debug_info: artifacts.debug_info, - mapped_process_debug_info_by_pid: artifacts.mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid: artifacts.mapped_process_unwind_data_by_pid, - mapped_process_module_symbols: artifacts.symbol_pid_mappings_by_pid, - path_key_to_path: artifacts.key_to_path, + artifacts: artifacts.artifacts, // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), ignored_modules: Default::default(), diff --git a/src/executor/wall_time/profiler/perf/parse_perf_file.rs b/src/executor/wall_time/profiler/perf/parse_perf_file.rs index 151b54945..1d1033b38 100644 --- a/src/executor/wall_time/profiler/perf/parse_perf_file.rs +++ b/src/executor/wall_time/profiler/perf/parse_perf_file.rs @@ -1,6 +1,6 @@ -use super::loaded_module::{LoadedModule, ProcessLoadedModule}; -use super::module_symbols::ModuleSymbols; -use super::unwind_data::unwind_data_from_elf; +use crate::executor::shared::module_artifacts::loaded_module::{LoadedModule, ProcessLoadedModule}; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; use libc::pid_t; use linux_perf_data::PerfFileReader; diff --git a/src/executor/wall_time/profiler/samply/mod.rs b/src/executor/wall_time/profiler/samply/mod.rs index 3d77e7ade..5f04ef8c8 100644 --- a/src/executor/wall_time/profiler/samply/mod.rs +++ b/src/executor/wall_time/profiler/samply/mod.rs @@ -184,11 +184,7 @@ impl Profiler for SamplyProfiler { // These fields aren't required in samply, since we symbolicate client-side. ignored_modules_by_pid: Default::default(), - debug_info: Default::default(), - mapped_process_debug_info_by_pid: Default::default(), - mapped_process_unwind_data_by_pid: Default::default(), - mapped_process_module_symbols: Default::default(), - path_key_to_path: Default::default(), + artifacts: Default::default(), // Deprecated fields below are no longer used debug_info_by_pid: Default::default(),