From eda0f01eed627942e0bac5f71621cb47a5454673 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:27 +0200 Subject: [PATCH 01/18] feat(memtrack): capture allocation stacks in eBPF Copy the caller's user stack in chunks at allocator entry and fold an FNV-1a digest over it in the kernel. The digest rides on the allocation event as stack_hash; the copied bytes, a DWARF-numbered register snapshot and a frame-pointer walk are emitted once per distinct digest on a dedicated ring buffer, so unwinding and symbolication can happen offline. Capture stays off until userspace sets the rodata toggle, so allocator probes are unchanged by default. Refs COD-3222 --- crates/memtrack/src/ebpf/c/allocator.h | 25 +- crates/memtrack/src/ebpf/c/event.h | 58 ++++- crates/memtrack/src/ebpf/c/main.bpf.c | 1 + .../memtrack/src/ebpf/c/stack_capture.bpf.h | 216 ++++++++++++++++++ .../memtrack/src/ebpf/c/utils/event_helpers.h | 21 +- .../memtrack/src/ebpf/c/utils/map_helpers.h | 8 + 6 files changed, 310 insertions(+), 19 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/stack_capture.bpf.h diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 9a4cc2387..8de96317f 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -9,6 +9,7 @@ BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ SEC(UPROBE_SEC) \ int uprobe_##name(struct pt_regs* ctx) { \ + stash_stack_hash(capture_stack(ctx)); \ return store_param(&name##_arg, arg_expr); \ } \ SEC(URETPROBE_SEC) \ @@ -17,6 +18,7 @@ if (!arg_ptr) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ __u64 ret_val = PT_REGS_RC(ctx); \ if (ret_val == 0) { \ return 0; \ @@ -32,6 +34,7 @@ if (arg0 == 0) { \ return 0; \ } \ + __u64 stack_hash = capture_stack(ctx); \ submit_block; \ } @@ -50,6 +53,8 @@ return 0; \ } \ \ + stash_stack_hash(capture_stack(ctx)); \ + \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY); \ @@ -63,6 +68,7 @@ if (!args) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ \ struct name##_args_t a = *args; \ bpf_map_delete_elem(&name##_args, &tid); \ @@ -77,20 +83,22 @@ submit_block; \ } -UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), + { return submit_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) +UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), - { return submit_calloc_event(arg0, ret_val); }) + { return submit_calloc_event(arg0, ret_val, stack_hash); }) UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx), - { return submit_realloc_event(arg1, ret_val, arg0); }) + { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); }) UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx), - { return submit_aligned_alloc_event(arg0, ret_val); }) + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) /* * posix_memalign(void** memptr, size_t alignment, size_t size) @@ -115,6 +123,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } + stash_stack_hash(capture_stack(ctx)); + struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); return 0; @@ -127,6 +137,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { if (!args) { return 0; } + __u64 stack_hash = take_stack_hash(); struct posix_memalign_args_t a = *args; bpf_map_delete_elem(&posix_memalign_args, &tid); @@ -140,7 +151,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - return submit_aligned_alloc_event(a.size, addr); + return submit_aligned_alloc_event(a.size, addr, stack_hash); } struct mmap_args { diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bf0677c93..7e10a7756 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -15,6 +15,49 @@ #define EVENT_TYPE_RSS 12 #define EVENT_TYPE_RMAP 13 +/* Largest user-stack copy one definition can carry. Every capture reserves + * header + budget in the ring up front, so this bounds ring space held per + * in-flight capture, per-capture copy cost, and the verifier work per load + * (which scales with the frozen budget). The kernel itself allows records up + * to the ring size; this is a policy cap. */ +#define MEMTRACK_MAX_STACK_COPY (32 * 1024) + +/* Registers, indexed by the capturing architecture's DWARF register number + * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30, + * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline + * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just + * ip/sp/bp. */ +#define MEMTRACK_STACK_REGS 33 + +/* Counter slots in the stack_counters array map. */ +#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 +#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 +/* bpf_get_stackid() has several negative outcomes (no user callchain, + * hash-bucket collision, or no free bucket), so this counts only missing ids. */ +#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2 +#define MEMTRACK_STACK_COUNTER_TRUNCATED 3 +#define MEMTRACK_STACK_COUNTER_RING_FULL 4 +#define MEMTRACK_STACK_COUNTER_COUNT 5 + +struct stack_regs { + uint64_t reg[MEMTRACK_STACK_REGS]; +}; + +/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp` + * follow it. */ +struct stack_header { + uint64_t hash; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ + uint32_t pid; + uint32_t tid; + uint32_t copy_len; + uint8_t truncated; /* the copy hit the size cap */ + uint8_t _pad[3]; + struct stack_regs regs; +}; + /* Common header shared by all event types */ struct event_header { uint8_t event_type; /* See EVENT_TYPE_* constants above */ @@ -29,20 +72,23 @@ struct event { union { /* Allocation events (malloc, calloc, aligned_alloc) */ struct { - uint64_t addr; /* address returned */ - uint64_t size; /* size requested */ + uint64_t addr; /* address returned */ + uint64_t size; /* size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } alloc; /* Deallocation event (free) */ struct { - uint64_t addr; /* address to free */ + uint64_t addr; /* address to free */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } free; /* Reallocation event - includes both old and new addresses */ struct { - uint64_t old_addr; /* previous address (can be NULL) */ - uint64_t new_addr; /* new address returned */ - uint64_t size; /* new size requested */ + uint64_t old_addr; /* previous address (can be NULL) */ + uint64_t new_addr; /* new address returned */ + uint64_t size; /* new size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } realloc; /* Memory mapping events (mmap, munmap, brk) */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 5a8d6ff07..b405f572b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -11,6 +11,7 @@ #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" +#include "stack_capture.bpf.h" #include "utils/event_helpers.h" #include "utils/folio.h" #include "utils/map_helpers.h" diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h new file mode 100644 index 000000000..438022ea4 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -0,0 +1,216 @@ +#ifndef __STACK_CAPTURE_BPF_H__ +#define __STACK_CAPTURE_BPF_H__ + +#include "event.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* Raw user-stack bytes and registers are emitted once per hash for offline + * DWARF unwinding. Allocation events carry the hash; bpf_get_stackid() supplies + * the frame-pointer fallback. + * + * Stack data changes may give one call path multiple hashes. Hashes may also + * recur after LRU eviction in seen_stack_hashes, so consumers must tolerate + * duplicate definitions. + */ + +const volatile __u8 capture_stacks_enabled = 0; +const volatile __u32 stack_copy_budget = 4096; + +#define STACK_TRACE_MAX_DEPTH 127 +/* Captured lengths are rounded down to this granularity. */ +#define STACK_COPY_CHUNK 512 +#define FNV64_OFFSET 0xcbf29ce484222325ULL +#define FNV64_PRIME 0x00000100000001b3ULL + +/* Frame-pointer fallback keyed by bpf_get_stackid(). */ +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 16384); + __type(key, __u32); + __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +/* A separate ring keeps allocation events fixed-size. */ +BPF_RINGBUF(stacks, 512 * 1024 * 1024); +BPF_LRU_HASH_MAP(seen_stack_hashes, __u64, __u8, 262144); +BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); +BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); + +static __always_inline void bump_stack_counter(__u32 index) { + __u64* slot = bpf_map_lookup_elem(&stack_counters, &index); + if (slot) { + __sync_fetch_and_add(slot, 1); + } +} + +/* 4-lane FNV-1a over one STACK_COPY_CHUNK worth of 8-byte words. Fixed-size, + * unrolled so the verifier sees a bounded loop. */ +static __always_inline void fnv64_hash_chunk(__u64 lanes[4], const __u64* words) { +#pragma unroll + for (__u32 i = 0; i < STACK_COPY_CHUNK / 8; i += 4) { + lanes[0] = (lanes[0] ^ words[i]) * FNV64_PRIME; + lanes[1] = (lanes[1] ^ words[i + 1]) * FNV64_PRIME; + lanes[2] = (lanes[2] ^ words[i + 2]) * FNV64_PRIME; + lanes[3] = (lanes[3] ^ words[i + 3]) * FNV64_PRIME; + } +} + +#if defined(__TARGET_ARCH_x86) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + out->reg[0] = ctx->ax; + out->reg[1] = ctx->dx; + out->reg[2] = ctx->cx; + out->reg[3] = ctx->bx; + out->reg[4] = ctx->si; + out->reg[5] = ctx->di; + out->reg[6] = ctx->bp; + out->reg[7] = ctx->sp; + out->reg[8] = ctx->r8; + out->reg[9] = ctx->r9; + out->reg[10] = ctx->r10; + out->reg[11] = ctx->r11; + out->reg[12] = ctx->r12; + out->reg[13] = ctx->r13; + out->reg[14] = ctx->r14; + out->reg[15] = ctx->r15; + out->reg[16] = ctx->ip; +} +#elif defined(__TARGET_ARCH_arm64) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + struct user_pt_regs* uregs = (struct user_pt_regs*)ctx; +#pragma unroll + for (int i = 0; i < 31; i++) { + out->reg[i] = uregs->regs[i]; + } + out->reg[31] = uregs->sp; + out->reg[32] = uregs->pc; +} +#else +#error "stack capture needs a DWARF register mapping for this architecture" +#endif + +static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { + void* slot = bpf_ringbuf_reserve(&stacks, sizeof(struct stack_header) + stack_copy_budget, 0); + if (!slot) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); + return 0; + } + + __u64 sp = PT_REGS_SP(ctx); + __u8* payload = (__u8*)slot + sizeof(struct stack_header); + __u64 lanes[4] = { + FNV64_OFFSET ^ 0, + FNV64_OFFSET ^ 1, + FNV64_OFFSET ^ 2, + FNV64_OFFSET ^ 3, + }; + __u32 got = 0; + + /* Chunked reads stop at the first unreadable stack region. + * Loop bound is checked against stack_copy_budget (a frozen rodata constant) + * so every slot access is provably in range. */ +#pragma clang loop unroll(disable) + for (__u32 off = 0; off + STACK_COPY_CHUNK <= stack_copy_budget; off += STACK_COPY_CHUNK) { + if (bpf_probe_read_user(payload + off, STACK_COPY_CHUNK, (void*)(sp + off)) != 0) { + break; + } + + fnv64_hash_chunk(lanes, (const __u64*)(payload + off)); + got = off + STACK_COPY_CHUNK; + } + + if (got == 0) { + bpf_ringbuf_discard(slot, 0); + bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); + return 0; + } + + __u8 truncated = got >= stack_copy_budget; + if (truncated) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); + } + + __u64 hash = + (((lanes[0] * FNV64_PRIME) ^ lanes[1]) * FNV64_PRIME ^ lanes[2]) * FNV64_PRIME ^ lanes[3]; + + /* Length distinguishes a full copy from the same bytes as a truncated prefix. + * Zero is reserved for allocation events without a stack. */ + hash = (hash ^ got) * FNV64_PRIME; + if (hash == 0) { + hash = FNV64_OFFSET; + } + + __u8 marker = 1; + long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); + if (gate_result == -17) { /* -EEXIST */ + bpf_ringbuf_discard(slot, 0); + return hash; + } + if (gate_result != 0) { + /* Re-emit when deduplication is full so the hash remains resolvable. */ + bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); + } + + __s64 stackid = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK); + if (stackid < 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_STACKID_FAILED); + } + + struct stack_header* header = (struct stack_header*)slot; + header->hash = hash; + header->timestamp = bpf_ktime_get_ns(); + header->stackid = stackid; + header->sp = sp; + header->pid = ids.tgid; + header->tid = ids.tid; + header->copy_len = got; + header->truncated = truncated; + header->_pad[0] = 0; + header->_pad[1] = 0; + header->_pad[2] = 0; + fill_stack_regs(&header->regs, ctx); + + bpf_ringbuf_submit(slot, 0); + return hash; +} + +static __always_inline __u64 capture_stack(struct pt_regs* ctx) { + if (!capture_stacks_enabled || !is_enabled()) { + return 0; + } + + struct task_ids ids = current_task_ids(); + if (!is_tracked(ids.tgid)) { + return 0; + } + + return capture_stack_inner(ctx, ids); +} + +static __always_inline void stash_stack_hash(__u64 hash) { + if (hash == 0) { + return; + } + + __u64 tid = current_tid(); + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); +} + +static __always_inline __u64 take_stack_hash(void) { + if (!capture_stacks_enabled) { + return 0; + } + + __u64 tid = current_tid(); + __u64* hash = bpf_map_lookup_elem(&pending_stack_hash, &tid); + if (!hash) { + return 0; + } + + __u64 value = *hash; + bpf_map_delete_elem(&pending_stack_hash, &tid); + return value; +} + +#endif /* __STACK_CAPTURE_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index ca5969a9a..ca53593d2 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -2,6 +2,7 @@ #define __EVENT_HELPERS_H__ #include "../event.h" +#include "../stack_capture.bpf.h" #include "map_helpers.h" #include "process_tracking.h" @@ -88,36 +89,44 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ } -static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); +static __always_inline int submit_free_event(__u64 addr, __u64 stack_hash) { + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { + e->data.free.addr = addr; + e->data.free.stack_hash = stack_hash; + }); } -static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { +static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, + __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; + e->data.realloc.stack_hash = stack_hash; }); } diff --git a/crates/memtrack/src/ebpf/c/utils/map_helpers.h b/crates/memtrack/src/ebpf/c/utils/map_helpers.h index 484fe9703..69422cdf9 100644 --- a/crates/memtrack/src/ebpf/c/utils/map_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/map_helpers.h @@ -9,6 +9,14 @@ __type(value, value_type); \ } name SEC(".maps") +#define BPF_LRU_HASH_MAP(name, key_type, value_type, max_ents) \ + struct { \ + __uint(type, BPF_MAP_TYPE_LRU_HASH); \ + __uint(max_entries, max_ents); \ + __type(key, key_type); \ + __type(value, value_type); \ + } name SEC(".maps") + #define BPF_ARRAY_MAP(name, value_type, max_ents) \ struct { \ __uint(type, BPF_MAP_TYPE_ARRAY); \ From e50662561f73d4263968f6924f72edd260ed9b10 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:53 +0200 Subject: [PATCH 02/18] feat(memtrack): add userspace stack-capture module Add the userspace half of allocation stack capture: env-driven configuration, stack-definition ring parsing, loss counters, per-pid module mapping tracking, a folding recorder that deduplicates definitions and counts occurrences, and the report it produces. Nothing constructs these yet; the tracker wiring follows. Refs COD-3222 --- Cargo.lock | 11 ++ crates/memtrack/src/ebpf/events.rs | 166 +++++++++++++++++- crates/memtrack/src/ebpf/mod.rs | 3 + crates/memtrack/src/ebpf/stacks.rs | 35 ++++ crates/memtrack/tests/c_tests.rs | 2 +- crates/memtrack/tests/dlopen_tests.rs | 11 +- crates/memtrack/tests/shared.rs | 7 +- crates/runner-shared/Cargo.toml | 5 + .../runner-shared/benches/memtrack_writer.rs | 160 ++++++++++++++--- .../src/artifacts/memtrack/mod.rs | 93 ++++++++-- .../src/artifacts/memtrack/pipeline.rs | 5 +- 11 files changed, 446 insertions(+), 52 deletions(-) create mode 100644 crates/memtrack/src/ebpf/stacks.rs diff --git a/Cargo.lock b/Cargo.lock index 1acec08f4..8e555476f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3616,6 +3616,7 @@ dependencies = [ "rmp", "rmp-serde", "serde", + "serde_bytes", "serde_json", "zstd", ] @@ -4062,6 +4063,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 4ed422a58..5a8abfce0 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -1,4 +1,6 @@ -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use crate::prelude::*; +use libbpf_rs::MapCore; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; // Include the bindings for event.h pub mod bindings { @@ -34,13 +36,20 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Malloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, + }, + ), + EVENT_TYPE_FREE => ( + event.data.free.addr, + MemtrackEventKind::Free { + stack_hash: event.data.free.stack_hash, }, ), - EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), EVENT_TYPE_CALLOC => ( event.data.alloc.addr, MemtrackEventKind::Calloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_REALLOC => ( @@ -48,12 +57,14 @@ pub fn parse_event(data: &[u8]) -> Option { MemtrackEventKind::Realloc { old_addr: Some(event.data.realloc.old_addr), size: event.data.realloc.size, + stack_hash: event.data.realloc.stack_hash, }, ), EVENT_TYPE_ALIGNED_ALLOC => ( event.data.alloc.addr, MemtrackEventKind::AlignedAlloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_MMAP => ( @@ -111,6 +122,74 @@ pub fn parse_event(data: &[u8]) -> Option { }) } +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { + warn!( + "malformed stack record: {} bytes, need at least {header_len}", + data.len() + ); + return None; + }; + + let record_len = header_len + header.copy_len as usize; + if data.len() < record_len { + warn!( + "malformed stack record: {} bytes, need {record_len}", + data.len() + ); + return None; + } + + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes: data[header_len..record_len].to_vec(), + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }), + }, + }; + + Some((event, header.stackid)) +} + +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + /// A request from the exec-mapping watcher to attach allocator probes. #[derive(Debug, Clone, Copy)] pub struct AttachRequest { @@ -157,6 +236,7 @@ mod tests { event.data.realloc.old_addr = 0x1000; event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; + event.data.realloc.stack_hash = 0xbeef; let bytes = event_bytes(&event); @@ -168,9 +248,14 @@ mod tests { assert_eq!(parsed.addr, 0x2000); match parsed.kind { - MemtrackEventKind::Realloc { old_addr, size } => { + MemtrackEventKind::Realloc { + old_addr, + size, + stack_hash, + } => { assert_eq!(old_addr, Some(0x1000)); assert_eq!(size, 256); + assert_eq!(stack_hash, 0xbeef); } _ => panic!("Expected Realloc event kind"), } @@ -186,6 +271,7 @@ mod tests { event.header.tid = 2000; event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; + event.data.alloc.stack_hash = 0x1234; let bytes = event_bytes(&event); @@ -197,8 +283,9 @@ mod tests { assert_eq!(parsed.addr, 0x1000); match parsed.kind { - MemtrackEventKind::Malloc { size } => { + MemtrackEventKind::Malloc { size, stack_hash } => { assert_eq!(size, 128); + assert_eq!(stack_hash, 0x1234); } _ => panic!("Expected Malloc event kind"), } @@ -277,3 +364,74 @@ mod tests { } } } + +#[cfg(test)] +mod stack_tests { + use super::*; + use crate::ebpf::events::bindings::stack_regs; + + fn encode(header: stack_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const stack_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + fn header(copy_len: u32) -> stack_header { + stack_header { + hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len, + truncated: 1, + _pad: [0; 3], + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); + let payload = [1, 2, 3, 4, 5]; + + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::Stack { record } = event.kind else { + panic!("expected Stack event"); + }; + + assert_eq!(record.hash, header.hash); + assert_eq!(record.sp, header.sp); + assert_eq!(record.regs, header.regs.reg.to_vec()); + assert_eq!(record.bytes, payload); + assert!(record.fp_chain.is_empty()); + assert!(record.truncated); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 2aa96549d..d964ebed5 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -4,9 +4,12 @@ mod memtrack; pub(crate) mod poller; mod proc_fs; mod spawn; +mod stacks; mod tracker; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; +pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; +pub use stacks::counters::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/stacks.rs b/crates/memtrack/src/ebpf/stacks.rs new file mode 100644 index 000000000..88605294b --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks.rs @@ -0,0 +1,35 @@ +use crate::ebpf::events::bindings::*; +use crate::prelude::*; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct StackCaptureStats { + pub copy_failed: u64, + pub hash_map_full: u64, + pub stackid_failed: u64, + pub truncated: u64, + pub ring_full: u64, +} + +impl StackCaptureStats { + pub fn read(map: &impl libbpf_rs::MapCore) -> Result { + Ok(Self { + copy_failed: slot(map, MEMTRACK_STACK_COUNTER_COPY_FAILED)?, + hash_map_full: slot(map, MEMTRACK_STACK_COUNTER_HASH_MAP_FULL)?, + stackid_failed: slot(map, MEMTRACK_STACK_COUNTER_STACKID_FAILED)?, + truncated: slot(map, MEMTRACK_STACK_COUNTER_TRUNCATED)?, + ring_full: slot(map, MEMTRACK_STACK_COUNTER_RING_FULL)?, + }) + } +} + +fn slot(map: &impl libbpf_rs::MapCore, index: u32) -> Result { + let value = map + .lookup(&index.to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read stack counter {index}"))? + .ok_or_else(|| anyhow!("stack counter slot {index} missing"))?; + let bytes: [u8; 8] = value + .as_slice() + .try_into() + .map_err(|_| anyhow!("stack counter {index} has unexpected size"))?; + Ok(u64::from_ne_bytes(bytes)) +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index db5fe6439..2d33dccf0 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -110,7 +110,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box Result<(), Box> { let malloc_addrs: HashSet = events .iter() .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size: 4242 } => Some(e.addr), + MemtrackEventKind::Malloc { size: 4242, .. } => Some(e.addr), _ => None, }) .collect(); let malloc_count = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let free_count = events .iter() .filter(|e| { - matches!(e.kind, MemtrackEventKind::Free) && malloc_addrs.contains(&e.addr) + matches!(e.kind, MemtrackEventKind::Free { .. }) + && malloc_addrs.contains(&e.addr) }) .count(); @@ -125,11 +126,11 @@ fn test_thread_dlopen() -> Result<(), Box> { |events| { let m4242 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let m4243 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243, .. })) .count(); assert_eq!(m4242, 100, "expected 100 mi_malloc(4242) events"); diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index c9e1c6267..342cbf6b7 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -31,7 +31,7 @@ macro_rules! assert_events_snapshot { matches!( e.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } @@ -108,7 +108,7 @@ pub fn between_markers(events: &[Event]) -> Vec { const MARKER: u64 = 0xC0D5_9EED; let is_marker = - |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size } if size == MARKER); + |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size, .. } if size == MARKER); events .iter() @@ -124,6 +124,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + ) }) .sorted_by_key(|e| e.timestamp) @@ -281,7 +282,7 @@ fn event_profile(events: &[Event]) -> EventProfile { if !matches!( event.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } diff --git a/crates/runner-shared/Cargo.toml b/crates/runner-shared/Cargo.toml index 8b8f6ab97..aee1707f0 100644 --- a/crates/runner-shared/Cargo.toml +++ b/crates/runner-shared/Cargo.toml @@ -4,9 +4,14 @@ publish = false version = "0.1.0" edition = "2024" +# Set by `cargo codspeed build` for the whole build; benches branch on it. +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(codspeed)'] } + [dependencies] anyhow = { workspace = true } serde = { workspace = true } +serde_bytes = "0.11" serde_json = { workspace = true } # Pinned to 1.x: 2.0 changes the wire format and serde integration bincode = "1.3" diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index a6c610e8e..1479fe96c 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -1,7 +1,22 @@ use divan::Bencher; +use divan::counter::{BytesCount, ItemsCount}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, MemtrackWriter, encode_events}; +use runner_shared::artifacts::{ + MemtrackEvent, MemtrackEventKind, MemtrackWriter, StackRecord, encode_events, +}; + +/// Reports allocation counts and bytes next to the timings for local runs. Only +/// tallies the thread running the benchmark, so the parallel encoder's +/// per-worker allocations show up in the single-threaded writer benches +/// instead. +/// +/// Left out of CodSpeed builds: wrapping the allocator costs ~15% on +/// allocation-heavy benchmarks, and the memory instrument reports allocations +/// there anyway. +#[cfg(not(codspeed))] +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); fn main() { divan::main(); @@ -14,14 +29,24 @@ fn generate_events(n: usize) -> Vec { for _ in 0..n { let size = rng.gen_range(8..8192); let kind = match rng.gen_range(0..10) { - 0 => MemtrackEventKind::Malloc { size }, - 1 => MemtrackEventKind::Free, + 0 => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, + 1 => MemtrackEventKind::Free { stack_hash: 0 }, 2 => MemtrackEventKind::Realloc { old_addr: Some(rng.r#gen()), size, + stack_hash: 0, + }, + 3 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, + 4 => MemtrackEventKind::AlignedAlloc { + size, + stack_hash: 0, }, - 3 => MemtrackEventKind::Calloc { size }, - 4 => MemtrackEventKind::AlignedAlloc { size }, 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, @@ -48,18 +73,25 @@ fn generate_events(n: usize) -> Vec { events } -#[divan::bench(args = [10_000, 100_000, 500_000, 1_000_000])] +/// Throughput of the single-threaded writer path: one zstd frame, no pool. +/// This is the per-worker ceiling the parallel encoder scales from. +#[divan::bench(args = [10_000, 100_000], max_time = 5.0)] fn write_events(bencher: Bencher, n: usize) { let events = generate_events(n); + let artifact_bytes = write_frame(&events).len(); + + bencher + .counter(ItemsCount::new(n)) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| write_frame(&events)); +} - bencher.bench_local(|| { - let mut output = Vec::new(); - let mut writer = MemtrackWriter::new(&mut output).unwrap(); - for event in &events { - writer.write_event(event).unwrap(); - } - writer.finish().unwrap(); - }); +fn write_frame(events: &[MemtrackEvent]) -> Vec { + let mut writer = MemtrackWriter::new(Vec::new()).unwrap(); + for event in events { + writer.write_event(event).unwrap(); + } + writer.finish().unwrap() } fn generate_realistic_events(n: usize) -> Vec { @@ -90,12 +122,18 @@ fn generate_realistic_events(n: usize) -> Vec { addr }); let kind = match rng.gen_range(0..20) { - 0 => MemtrackEventKind::Calloc { size }, + 0 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, 1 => MemtrackEventKind::Mmap { size }, - _ => MemtrackEventKind::Malloc { size }, + _ => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, }; - if let MemtrackEventKind::Mmap { size } = kind { - live_mmap.push((addr, size)); + if let MemtrackEventKind::Mmap { size } = &kind { + live_mmap.push((addr, *size)); } else { live_heap.push(addr); } @@ -105,7 +143,7 @@ fn generate_realistic_events(n: usize) -> Vec { if idx < live_heap.len() { let addr = live_heap.swap_remove(idx); free_list.push(addr); - (addr, MemtrackEventKind::Free) + (addr, MemtrackEventKind::Free { stack_hash: 0 }) } else { let (addr, size) = live_mmap.swap_remove(idx - live_heap.len()); free_list.push(addr); @@ -127,6 +165,7 @@ fn generate_realistic_events(n: usize) -> Vec { MemtrackEventKind::Realloc { old_addr: Some(old_addr), size, + stack_hash: 0, }, ) }; @@ -142,15 +181,84 @@ fn generate_realistic_events(n: usize) -> Vec { events } -const REALISTIC_EVENTS: usize = 1_000_000; +/// One event per frame slot of a full window, so every worker count from 1 to +/// `WINDOW_FRAMES` has a frame to take. Sizing below this hides pool scaling: +/// the encoder can only parallelize across whole frames. +const REALISTIC_EVENTS: usize = 16 * 64 * 1024; -#[divan::bench(args = [16, 8, 4], max_time = 10.0)] +/// Throughput of the artifact encoder over a realistic allocation mix, as a +/// function of the worker pool size. +#[divan::bench(args = [1, 2, 4, 8, 16], max_time = 10.0)] fn encode_events_realistic(bencher: Bencher, n_workers: usize) { let events = generate_realistic_events(REALISTIC_EVENTS); + let artifact_bytes = encode(&events, n_workers).len(); - bencher.bench_local(|| { - let mut output = Vec::new(); - encode_events(events.iter().copied(), &mut output, n_workers).unwrap(); - output - }); + bencher + .counter(ItemsCount::new(events.len())) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| encode(&events, n_workers)); +} + +/// Single-frame throughput on captured stacks: each `Stack` event carries a +/// register set and a raw stack copy, so payloads are orders of magnitude +/// larger than an allocation record and the byte rate is what matters. +#[divan::bench(max_time = 10.0)] +fn write_stack_events(bencher: Bencher) { + let events = generate_stack_events(16 * 1024); + let artifact_bytes = write_frame(&events).len(); + + bencher + .counter(ItemsCount::new(events.len())) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| write_frame(&events)); +} + +fn encode(events: &[MemtrackEvent], n_workers: usize) -> Vec { + let mut output = Vec::new(); + encode_events(events.iter().cloned(), &mut output, n_workers).unwrap(); + output +} + +/// A stack-capture heavy stream: one `Stack` record per allocation, sized like +/// the kernel's stack copies (2 KiB payload, x86_64 register set). +fn generate_stack_events(n: usize) -> Vec { + const STACK_BYTES: usize = 2048; + let mut rng = StdRng::seed_from_u64(7); + let mut events = Vec::with_capacity(n * 2); + + while events.len() < n * 2 { + let hash: u64 = rng.r#gen(); + let record = StackRecord { + hash, + sp: 0x7fff_0000_0000 | (rng.gen_range(0..1u64 << 20) << 4), + regs: (0..33).map(|_| rng.r#gen()).collect(), + bytes: (0..STACK_BYTES).map(|_| rng.r#gen()).collect(), + fp_chain: (0..16).map(|_| rng.r#gen()).collect(), + truncated: false, + }; + let addr: u64 = rng.r#gen(); + let timestamp: u64 = rng.r#gen(); + + events.push(MemtrackEvent { + pid: 4242, + tid: 4242, + timestamp, + addr, + kind: MemtrackEventKind::Stack { + record: Box::new(record), + }, + }); + events.push(MemtrackEvent { + pid: 4242, + tid: 4242, + timestamp: timestamp + 1, + addr, + kind: MemtrackEventKind::Malloc { + size: rng.gen_range(8..8192), + stack_hash: hash, + }, + }); + } + + events } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b082a7c67..433aab5a5 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -41,7 +41,7 @@ impl MemtrackArtifact { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct MemtrackEvent { pub pid: pid_t, pub tid: pid_t, @@ -51,23 +51,34 @@ pub struct MemtrackEvent { pub kind: MemtrackEventKind, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] pub enum MemtrackEventKind { Malloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, + }, + Free { + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, - Free, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Calloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, AlignedAlloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Mmap { size: u64, @@ -91,6 +102,31 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + + Stack { + #[serde(flatten)] + record: Box, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StackRecord { + pub hash: u64, + /// User stack pointer the copy starts at. + pub sp: u64, + /// Registers by DWARF number for the capturing architecture; 33 entries on x86_64. + pub regs: Vec, + /// Raw stack bytes read upward from `sp`. + #[serde(with = "serde_bytes")] + pub bytes: Vec, + /// In-kernel frame-pointer walk, innermost first; empty when unavailable. + pub fp_chain: Vec, + /// The copy filled its budget, so stack above it was not captured. + pub truncated: bool, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 } pub struct MemtrackEventStream { @@ -120,14 +156,17 @@ mod tests { tid: 11, timestamp: 100, addr: 0x10, - kind: MemtrackEventKind::Malloc { size: 64 }, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, }, MemtrackEvent { pid: 1, tid: 12, timestamp: 200, addr: 0x20, - kind: MemtrackEventKind::Free, + kind: MemtrackEventKind::Free { stack_hash: 0 }, }, MemtrackEvent { pid: 1, @@ -167,21 +206,47 @@ mod tests { } let kinds = [ - MemtrackEventKind::Malloc { size: 7 }, - MemtrackEventKind::Free, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0, + }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0xCAFE_BABE, + }, + MemtrackEventKind::Free { stack_hash: 0 }, + MemtrackEventKind::Free { stack_hash: 0xFEED }, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, + stack_hash: 0, }, MemtrackEventKind::Realloc { old_addr: None, size: 42, + stack_hash: 0x1234, + }, + MemtrackEventKind::Calloc { + size: 9, + stack_hash: 0, + }, + MemtrackEventKind::AlignedAlloc { + size: 9, + stack_hash: 0, }, - MemtrackEventKind::Calloc { size: 9 }, - MemtrackEventKind::AlignedAlloc { size: 9 }, MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: 0xDEAD_BEEF, + sp: 0x7FFF_0000, + regs: vec![0; 33], + bytes: vec![1, 2, 3, 4], + fp_chain: vec![0x1000, 0x2000], + truncated: false, + }), + }, ]; for kind in kinds { @@ -190,7 +255,7 @@ mod tests { tid: 42, timestamp: 0xDEAD, addr: 0xBEEF, - kind, + kind: kind.clone(), }; let shadow = Shadow { pid: -7, @@ -215,7 +280,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect(); @@ -265,7 +333,8 @@ mod tests { event.kind, MemtrackEventKind::Realloc { old_addr: None, - size: 42 + size: 42, + stack_hash: 0, } )); diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index c47b3aed9..8cac46f05 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -94,7 +94,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect() } From 239cad78be9e202e5653e171524d456b4e42f0de Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 11:33:08 +0200 Subject: [PATCH 03/18] feat(memtrack): enable stack capture through the tracker Wire the capture rodata and map sizing into skeleton load, poll the stack-definition ring alongside the event ring, and expose the loss counters and frame-pointer chains. The attach worker snapshots module mappings while it holds a process stopped, which is the only point they are guaranteed readable. Guard the lifecycle: finishing with a live session would block forever on the recorder, and a second spawn would leave the capture rings undrained, so both now fail with a descriptive error. With capture disabled the ring buffer and frame-pointer map shrink to the allocator minimum rather than reserving tens of MiB. Refs COD-3222 --- crates/memtrack/src/ebpf/memtrack/maps.rs | 5 ++ crates/memtrack/src/ebpf/memtrack/mod.rs | 57 +++++++++++++++++++++-- crates/memtrack/src/ebpf/tracker.rs | 47 ++++++++++++++++--- crates/memtrack/src/main.rs | 7 +++ crates/memtrack/src/session.rs | 3 ++ 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index c7376d463..994650d53 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -1,4 +1,5 @@ use super::MemtrackBpf; +use crate::ebpf::stacks::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; @@ -68,6 +69,10 @@ impl MemtrackBpf { ) } + pub fn stack_capture_stats(&self) -> Result { + StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) + } + pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 4c32a33a0..44705cc25 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -123,19 +123,25 @@ pub struct MemtrackBpf { impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool) -> Result { + pub fn new_with_rmap(track_rmap: bool, stack_copy_size: Option) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap) + Self::with_variant(variant, track_rmap, stack_copy_size) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. - pub fn with_variant(variant: BpfVariant, track_rmap: bool) -> Result { + /// + /// `stack_copy_size` turns on allocation stack capture. + pub fn with_variant( + variant: BpfVariant, + track_rmap: bool, + stack_copy_size: Option, + ) -> Result { crate::kernel::KernelBtf::ensure_available()?; let page_shift = page_shift()?; @@ -166,6 +172,19 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } + if let Some(copy_size) = stack_copy_size { + rodata.capture_stacks_enabled = 1; + rodata.stack_copy_size = copy_size; + } + } + + // Avoid reserving the stack maps when capture is disabled. A + // ring buffer's size must stay a power-of-two page count. + if stack_copy_size.is_none() { + open_skel.maps.stacks.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; } // Autoload is decided before load(), so fentries whose targets @@ -229,6 +248,38 @@ impl MemtrackBpf { )) } + /// Poll the stack-record ring buffer into `tx`. + pub(crate) fn poll_stacks( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + use crate::ebpf::stacks::events; + use runner_shared::artifacts::MemtrackEventKind; + + // The poller outlives this borrow of the skeleton, so the chain lookup + // needs an owned handle rather than a reference to the skeleton map. + let stack_traces = with_skel!(self, skel => { + libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) + .context("Failed to create handle for stack_traces map")? + }); + + let parse = move |data: &[u8]| { + let (mut event, stackid) = events::parse_stack(data)?; + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + Some(event) + }; + + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.stacks, + parse, + tx, + poll_interval_ms, + )) + } + /// Poll the exec-mapping request ring buffer into `tx`. Same contract as /// [`Self::poll_events_with_channel`]. pub(crate) fn poll_attach_with_channel( diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index dc9bc13ef..5536e5324 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,5 +1,7 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; +use crate::ebpf::stacks::config::{clamp_copy_size, stack_copy_size_from_env}; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; @@ -7,6 +9,7 @@ use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; @@ -19,6 +22,10 @@ pub struct TrackerOptions { /// Reconstruct per-process RSS from the folio rmap fentry hooks. #[builder(default = false)] pub rmap: bool, + /// Bytes of user stack to copy for each allocation event. `None` leaves + /// stack capture off; values are clamped to the supported range. + #[builder(default = None)] + pub stack_copy_size: Option, } impl TrackerOptions { @@ -29,6 +36,7 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .rmap(std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1")) + .stack_copy_size(stack_copy_size_from_env()) .build() } } @@ -37,6 +45,9 @@ pub struct Tracker { bpf: Arc>, worker: Mutex>, allocators: bool, + /// The dedup gate spans the whole BPF object, so a second session would + /// reference stack records the first one already consumed. + stacks_polled: Option, } impl Tracker { @@ -48,9 +59,11 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { + let copy_size = options.stack_copy_size.map(clamp_copy_size); Self::build( - MemtrackBpf::new_with_rmap(options.rmap)?, + MemtrackBpf::new_with_rmap(options.rmap, copy_size)?, options.allocators, + copy_size.is_some(), ) } @@ -58,13 +71,17 @@ impl Tracker { /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = TrackerOptions::from_env().rmap; - Self::build(MemtrackBpf::with_variant(variant, track_rmap)?, true) + Self::build( + MemtrackBpf::with_variant(variant, track_rmap, None)?, + true, + false, + ) } /// Build a tracker: attach lifetime tracepoints (and rmap fentries when the /// skeleton was opened for them), plus, when `allocators` is set, the /// exec-mapping watcher and the on-demand allocator-attach worker. - fn build(mut bpf: MemtrackBpf, allocators: bool) -> Result { + fn build(mut bpf: MemtrackBpf, allocators: bool, capture_stacks: bool) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; @@ -83,6 +100,7 @@ impl Tracker { bpf, worker: Mutex::new(worker), allocators, + stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), }) } @@ -95,6 +113,14 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { + let capture_stacks = match &self.stacks_polled { + Some(polled) if polled.swap(true, Ordering::Relaxed) => { + bail!("stack capture supports a single spawned command per tracker") + } + Some(_) => true, + None => false, + }; + let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); @@ -102,6 +128,7 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; + match self.worker.lock().as_ref() { Some(worker) => worker.set_root_pid(pid), // No watcher to arm means exec mappings would be missed. @@ -110,14 +137,17 @@ impl Tracker { } let (tx, rx) = mpsc::channel(); - let poller = { + let (poller, stack_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; - bpf.poll_events_with_channel(10, tx)? + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx)?, stack_poller) }; resume(pid)?; - Ok(Session::new(child, rx, poller)) + Ok(Session::new(child, rx, poller, stack_poller)) } /// Enable allocator-event tracking in the BPF program. Lifetime events @@ -138,6 +168,11 @@ impl Tracker { self.bpf.lock().dropped_events_count() } + /// Per-cause counts of stack captures that were skipped or truncated. + pub fn stack_capture_stats(&self) -> Result { + self.bpf.lock().stack_capture_stats() + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 283cff194..d9a6411b1 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,6 +159,13 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; + if tracker.stack_capture_enabled() { + let stats = tracker + .stack_capture_stats() + .context("Failed to read stack capture stats")?; + debug!("stack capture stats: {stats:?}"); + } + // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 7aec33feb..9bed66b7d 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -10,6 +10,7 @@ pub struct Session { child: Child, events: Option>, _poller: RingBufferPoller, + _stack_poller: Option, } impl Session { @@ -17,11 +18,13 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, + stack_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, + _stack_poller: stack_poller, } } From 9929b053af463e1762ecfc598eeec06dc98822c0 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:01:18 +0200 Subject: [PATCH 04/18] test(memtrack): cover allocation stack capture Add a fixture with two non-inlinable malloc call paths and privileged tests over it: distinct call paths get distinct identities with module mappings for the binary and libc, repeated calls deduplicate, and the default-off path still reports allocations. Two cases guard failure modes the default budget cannot reach. The maximum copy budget is the only configuration that exercises the verifier's instruction limit, since the frozen rodata makes the copy and hash loops scale with the configured size. Shrinking the frame-pointer map to one slot proves exhaustion costs only the fallback chain, never an allocation event. Refs COD-3222 --- .github/workflows/ci.yml | 2 +- crates/memtrack/testdata/stack_paths.c | 46 ++++ crates/memtrack/tests/c_tests.rs | 2 +- crates/memtrack/tests/rss_tests.rs | 24 +- crates/memtrack/tests/shared.rs | 21 +- .../stack_tests__nested_doubling.snap | 12 + ...ck_tests__nested_doubling_shared_free.snap | 12 + .../stack_tests__stack_capture_disabled.snap | 206 +++++++++++++++ .../snapshots/stack_tests__stack_paths.snap | 206 +++++++++++++++ crates/memtrack/tests/stack_tests.rs | 240 ++++++++++++++++++ 10 files changed, 758 insertions(+), 13 deletions(-) create mode 100644 crates/memtrack/testdata/stack_paths.c create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap create mode 100644 crates/memtrack/tests/stack_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75eac..3265a9194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/crates/memtrack/testdata/stack_paths.c b/crates/memtrack/testdata/stack_paths.c new file mode 100644 index 000000000..0cee7f437 --- /dev/null +++ b/crates/memtrack/testdata/stack_paths.c @@ -0,0 +1,46 @@ +#include +#include + +static volatile void *escaped_pointer; +static volatile unsigned int remaining_a = 50; +static volatile unsigned int remaining_b = 50; + +__attribute__((noinline)) static void path_a_inner(void) { + void *pointer = malloc(64); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_a(void) { + while (remaining_a != 0) { + path_a_inner(); + --remaining_a; + } +} + +__attribute__((noinline)) static void path_b_inner(void) { + void *pointer = malloc(192); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_b(void) { + while (remaining_b != 0) { + path_b_inner(); + --remaining_b; + } +} + +int main(void) { + void *marker_before = malloc(0xC0D59EED); + escaped_pointer = marker_before; + free(marker_before); + + path_a(); + path_b(); + + void *marker_after = malloc(0xC0D59EED); + escaped_pointer = marker_after; + free(marker_after); + return 0; +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index 2d33dccf0..d217ff392 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -89,7 +89,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box TrackerOptions { + TrackerOptions::builder() + .allocators(false) + .rmap(true) + .build() +} + /// Run a fixture under `track` and return the raw `/proc` RSS report it wrote to /// its argv[1] alongside the collected events. /// @@ -349,7 +357,9 @@ fn test_rss_rmap_tracking( #[case] source: &str, #[case] name: &str, ) -> Result<(), Box> { - let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let (raw_report, events) = track_fixture(source, name, |command| { + shared::track_command(command, rmap_only_options()) + })?; let raw_report = raw_report.ok_or("fixture wrote no rss report")?; let (rss_stat, rmap) = per_pid_peaks(&events); let summary = RssSummary { @@ -425,14 +435,14 @@ enum Reclaim { #[case::rss_stat(Reclaim::RssStat)] #[case::rmap(Reclaim::Rmap)] fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box> { - let track: fn(Command) -> shared::TrackResult = match mode { - Reclaim::RssStat => shared::track_command, - Reclaim::Rmap => shared::track_command_with_rmap, + let options = match mode { + Reclaim::RssStat => TrackerOptions::default(), + Reclaim::Rmap => rmap_only_options(), }; let (_report, events) = track_fixture( include_str!("../testdata/rss/madvise_extern.c"), "madvise_extern", - track, + |command| shared::track_command(command, options), )?; // A = owner that faulted the file region; B = external caller, single-threaded @@ -601,7 +611,7 @@ fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box Result<(), Box String { match kind { + MemtrackEventKind::Free { .. } => "Free".to_string(), + MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), + MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), + MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), MemtrackEventKind::Realloc { size, .. } => format!("Realloc {{ size: {size} }}"), other => format!("{other:?}"), } @@ -124,7 +127,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit - + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -238,6 +241,16 @@ pub fn track_command_with_rmap_maps( Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } +/// Track a command with allocation stack capture enabled, returning its events. +pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { + track_command_with_opts( + command, + TrackerOptions::builder() + .stack_copy_size(Some(copy_size)) + .build(), + ) +} + /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. /// diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap new file mode 100644 index 000000000..cf9fc935d --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap new file mode 100644 index 000000000..7cb31d31e --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs new file mode 100644 index 000000000..7a95cd6ca --- /dev/null +++ b/crates/memtrack/tests/stack_tests.rs @@ -0,0 +1,240 @@ +#[macro_use] +mod shared; + +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::collections::HashSet; +use std::process::Command; +use tempfile::TempDir; + +const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; + +fn compile_fixture( + name: &str, + temp_dir: &TempDir, +) -> Result> { + shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + name, + temp_dir.path(), + ) +} +fn require_mapping_support() -> bool { + if memtrack::MappingSupport::detect() == memtrack::MappingSupport::Unsupported { + eprintln!("skipping stack capture test: mapping support is unavailable"); + return false; + } + true +} + +/// The stack identity carried by each allocation and deallocation event that has one. +fn event_hashes(events: &[MemtrackEvent]) -> Vec { + events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } + | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), + _ => None, + }) + .collect() +} + +fn record_hashes(events: &[MemtrackEvent]) -> HashSet { + events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record } => Some(record.hash), + _ => None, + }) + .collect() +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let records: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } => { + Some((r.hash, r.sp, &r.regs, &r.bytes, r.truncated)) + } + _ => None, + }) + .collect(); + + assert!( + records.len() >= 2, + "expected at least two stack records, got {} ({} events)", + records.len(), + events.len() + ); + + let hashes = record_hashes(&events); + assert_eq!( + hashes.len(), + records.len(), + "stack records must be deduplicated by unique hash" + ); + + for (hash, sp, regs, bytes, truncated) in &records { + assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); + assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); + assert!( + !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, + "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", + bytes.len() + ); + assert_eq!( + *truncated, + bytes.len() == COPY_SIZE as usize, + "record {hash:#x} may only be flagged truncated when it filled the budget" + ); + } + + let carried = event_hashes(&events); + assert!( + !carried.is_empty(), + "expected events carrying a captured stack hash" + ); + assert!( + carried.iter().all(|hash| hashes.contains(hash)), + "every non-zero stack_hash must have a matching stack record" + ); + + // The fixture frees every allocation, so both sides must report identities. + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), + "free events must carry their own stack identity" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let carried = event_hashes(&events); + let records = record_hashes(&events); + assert!( + carried.len() > records.len(), + "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", + carried.len(), + records.len(), + events.len() + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// The largest budget stresses the verifier hardest: the copy loop and its +/// unrolled per-chunk hash both scale with the configured size, so a program +/// that loads at the default can still exceed the instruction limit here. +/// It is also the only budget at which nothing can be budget-limited, because +/// the stack mapping always ends first. +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn max_copy_budget_loads_and_captures_whole_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_max", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; + + let truncated: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } if r.truncated => Some(r.hash), + _ => None, + }) + .collect(); + + assert!( + !record_hashes(&events).is_empty(), + "expected stack records at the maximum copy budget" + ); + assert!( + truncated.is_empty(), + "no capture can be budget-limited at the maximum budget: {truncated:#x?}" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// Restores the capture toggle on drop so a failing assertion cannot leak the +/// override into later tests (the suite runs single-threaded). +struct DisableCaptureGuard; + +impl DisableCaptureGuard { + fn set() -> Self { + // SAFETY: tests run with --test-threads 1, so no concurrent env access. + unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; + Self + } +} + +impl Drop for DisableCaptureGuard { + fn drop(&mut self) { + // SAFETY: see `set`. + unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; + } +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; + let _guard = DisableCaptureGuard::set(); + let (events, thread_handle) = shared::track_binary(&binary)?; + + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), + "disabled capture must still report allocation events" + ); + assert!( + record_hashes(&events).is_empty(), + "disabled capture must emit zero stack records" + ); + assert!( + event_hashes(&events).is_empty(), + "disabled capture must leave stack_hash zero on every event" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} From 1b599805323cf9bc453ef5171db8f60e98de857d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 19:44:42 +0200 Subject: [PATCH 05/18] refactor(runner): move ELF artifact pipeline to executor/shared The symbol, unwind-data and debug-info extraction is not perf-specific: it turns a set of mapped ELF modules into the deduplicated keyed artifacts a profile references, whatever discovered the mappings. Memory mode needs the same pipeline, so it moves out of wall_time/profiler/perf into executor/shared/module_artifacts. --- src/executor/shared/mod.rs | 1 + .../module_artifacts}/debug_info.rs | 0 .../module_artifacts}/elf_helper.rs | 0 .../module_artifacts}/loaded_module.rs | 0 src/executor/shared/module_artifacts/mod.rs | 15 +++++++++++++++ .../module_artifacts}/module_symbols.rs | 0 .../perf => shared/module_artifacts}/naming.rs | 0 .../module_artifacts}/save_artifacts.rs | 2 +- ...facts__debug_info__tests__cpp_debug_info.snap} | 2 +- ...ts__debug_info__tests__golang_debug_info.snap} | 2 +- ...acts__debug_info__tests__ruff_debug_info.snap} | 2 +- ...debug_info__tests__rust_divan_debug_info.snap} | 2 +- ...g_info__tests__the_algorithms_debug_info.snap} | 2 +- ...acts__module_symbols__tests__cpp_symbols.snap} | 2 +- ...s__module_symbols__tests__golang_symbols.snap} | 2 +- ...cts__module_symbols__tests__ruff_symbols.snap} | 2 +- ...odule_symbols__tests__rust_divan_symbols.snap} | 2 +- ...e_symbols__tests__the_algorithms_symbols.snap} | 2 +- ...cts__unwind_data__tests__cpp_unwind_data.snap} | 2 +- ...__unwind_data__tests__golang_unwind_data.snap} | 2 +- ...ts__unwind_data__tests__ruff_unwind_data.snap} | 2 +- ...wind_data__tests__rust_divan_unwind_data.snap} | 2 +- ..._data__tests__the_algorithms_unwind_data.snap} | 2 +- .../module_artifacts}/unwind_data.rs | 0 src/executor/wall_time/profiler/perf/jit_dump.rs | 2 +- src/executor/wall_time/profiler/perf/mod.rs | 8 +------- .../wall_time/profiler/perf/parse_perf_file.rs | 6 +++--- 27 files changed, 37 insertions(+), 27 deletions(-) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/debug_info.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/elf_helper.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/loaded_module.rs (100%) create mode 100644 src/executor/shared/module_artifacts/mod.rs rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/module_symbols.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/naming.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/save_artifacts.rs (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/unwind_data.rs (100%) diff --git a/src/executor/shared/mod.rs b/src/executor/shared/mod.rs index 2badf4064..f278f07cd 100644 --- a/src/executor/shared/mod.rs +++ b/src/executor/shared/mod.rs @@ -1 +1,2 @@ pub mod fifo; +pub mod module_artifacts; diff --git a/src/executor/wall_time/profiler/perf/debug_info.rs b/src/executor/shared/module_artifacts/debug_info.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/debug_info.rs rename to src/executor/shared/module_artifacts/debug_info.rs diff --git a/src/executor/wall_time/profiler/perf/elf_helper.rs b/src/executor/shared/module_artifacts/elf_helper.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/elf_helper.rs rename to src/executor/shared/module_artifacts/elf_helper.rs diff --git a/src/executor/wall_time/profiler/perf/loaded_module.rs b/src/executor/shared/module_artifacts/loaded_module.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/loaded_module.rs rename to src/executor/shared/module_artifacts/loaded_module.rs diff --git a/src/executor/shared/module_artifacts/mod.rs b/src/executor/shared/module_artifacts/mod.rs new file mode 100644 index 000000000..72c2bf0ea --- /dev/null +++ b/src/executor/shared/module_artifacts/mod.rs @@ -0,0 +1,15 @@ +//! Extraction of symbols, unwind data and debug info from the ELF modules a +//! profiled process mapped, and their deduplicated on-disk layout. +//! +//! The input is a set of [`loaded_module::LoadedModule`]s, however the mappings +//! were discovered; the output is the keyed `unwind_data`/`symbols.map` files +//! plus the per-pid references that the metadata points at. + +mod elf_helper; +mod naming; + +pub mod debug_info; +pub mod loaded_module; +pub mod module_symbols; +pub mod save_artifacts; +pub mod unwind_data; diff --git a/src/executor/wall_time/profiler/perf/module_symbols.rs b/src/executor/shared/module_artifacts/module_symbols.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/module_symbols.rs rename to src/executor/shared/module_artifacts/module_symbols.rs diff --git a/src/executor/wall_time/profiler/perf/naming.rs b/src/executor/shared/module_artifacts/naming.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/naming.rs rename to src/executor/shared/module_artifacts/naming.rs diff --git a/src/executor/wall_time/profiler/perf/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs similarity index 99% rename from src/executor/wall_time/profiler/perf/save_artifacts.rs rename to src/executor/shared/module_artifacts/save_artifacts.rs index 36b2fd12a..9b4899427 100644 --- a/src/executor/wall_time/profiler/perf/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -1,7 +1,7 @@ use super::debug_info::debug_info_by_path; use super::loaded_module::LoadedModule; +use super::naming; use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; -use crate::executor::wall_time::profiler::perf::naming; use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap index 48d654070..9b917e545 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap index e92dcefa8..5b6a04ae2 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap index 75d0a4494..97dc7b43a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap index 6cf90c6a1..fd5e1aa0c 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap index 9e9c52a2e..a238cbb28 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap index 8456dd05b..34c79e04e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap index 84138e7bb..990e660d9 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap index 879d29f90..fe5907dd0 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap index 839039f35..10a77b716 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap index fec3e2802..724f3002e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap index 205b5e148..6c554024a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap index 699e4b031..807e10611 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap index a0a5b0f98..3956fd7d1 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap index 0367c2dee..edfd8e558 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap index 9fc15dca2..066c1ad0e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/unwind_data.rs b/src/executor/shared/module_artifacts/unwind_data.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/unwind_data.rs rename to src/executor/shared/module_artifacts/unwind_data.rs diff --git a/src/executor/wall_time/profiler/perf/jit_dump.rs b/src/executor/wall_time/profiler/perf/jit_dump.rs index fd5fad056..344f4080e 100644 --- a/src/executor/wall_time/profiler/perf/jit_dump.rs +++ b/src/executor/wall_time/profiler/perf/jit_dump.rs @@ -1,4 +1,4 @@ -use super::module_symbols::{ModuleSymbols, Symbol}; +use crate::executor::shared::module_artifacts::module_symbols::{ModuleSymbols, Symbol}; use crate::prelude::*; use linux_perf_data::jitdump::{JitDumpReader, JitDumpRecord}; use runner_shared::unwind_data::{ProcessUnwindData, UnwindData}; diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 8816e5fb0..625407e86 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -10,6 +10,7 @@ use crate::executor::helpers::env::suppress_go_perf_unwinding_warning; use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; +use crate::executor::shared::module_artifacts::save_artifacts; use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING; use crate::executor::wall_time::profiler::Profiler; use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ; @@ -29,16 +30,9 @@ use runner_shared::metadata::WalltimeMetadata; use std::path::Path; use std::path::PathBuf; -mod debug_info; -mod elf_helper; mod jit_dump; -mod loaded_module; -mod module_symbols; -mod naming; mod parse_perf_file; -mod save_artifacts; pub(crate) mod setup; -mod unwind_data; pub mod fifo; pub mod perf_executable; diff --git a/src/executor/wall_time/profiler/perf/parse_perf_file.rs b/src/executor/wall_time/profiler/perf/parse_perf_file.rs index 151b54945..1d1033b38 100644 --- a/src/executor/wall_time/profiler/perf/parse_perf_file.rs +++ b/src/executor/wall_time/profiler/perf/parse_perf_file.rs @@ -1,6 +1,6 @@ -use super::loaded_module::{LoadedModule, ProcessLoadedModule}; -use super::module_symbols::ModuleSymbols; -use super::unwind_data::unwind_data_from_elf; +use crate::executor::shared::module_artifacts::loaded_module::{LoadedModule, ProcessLoadedModule}; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; use libc::pid_t; use linux_perf_data::PerfFileReader; From b8083aa639e2dc80878ee158ac57527bd803ce1a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 19:53:32 +0200 Subject: [PATCH 06/18] feat(runner): add MemtrackMetadata sharing ModuleArtifacts with walltime Memory mode needs the same per-pid module references walltime writes, so the five artifact fields move into a flattened `ModuleArtifacts` shared by both formats; walltime's JSON is unchanged, asserted against output captured from the flat struct. Flattening buffers those fields through serde's `Content`, which unlike serde_json's direct deserializer cannot parse a string JSON key into a pid, so pid-keyed maps get an explicit key-parsing helper. --- crates/runner-shared/src/lib.rs | 1 + crates/runner-shared/src/metadata.rs | 191 ++++++++++++++++-- crates/runner-shared/src/serde_pid_map.rs | 36 ++++ .../shared/module_artifacts/save_artifacts.rs | 25 +-- src/executor/wall_time/profiler/perf/mod.rs | 6 +- src/executor/wall_time/profiler/samply/mod.rs | 6 +- 6 files changed, 229 insertions(+), 36 deletions(-) create mode 100644 crates/runner-shared/src/serde_pid_map.rs diff --git a/crates/runner-shared/src/lib.rs b/crates/runner-shared/src/lib.rs index 61e804de7..2cdc7d5a6 100644 --- a/crates/runner-shared/src/lib.rs +++ b/crates/runner-shared/src/lib.rs @@ -4,5 +4,6 @@ pub mod fifo; pub mod metadata; pub mod module_symbols; pub mod perf_event; +pub mod serde_pid_map; pub mod unwind_data; pub mod walltime_results; diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 7a2c7c89b..654ae298d 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -11,35 +11,43 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; +/// The per-profile module artifacts: the deduplicated debug info, unwind data +/// and symbol tables extracted from the ELF modules the profiled processes +/// mapped, plus the per-pid references into them. +/// +/// Flattened into every metadata format, so all profiling modes describe their +/// modules identically. #[derive(Serialize, Deserialize, Default)] -pub struct WalltimeMetadata { - /// The version of this metadata format. - pub version: u64, - - /// Name and version of the integration - pub integration: (String, String), - - /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub ignored_modules_by_pid: HashMap>, - +pub struct ModuleArtifacts { /// Deduplicated debug info entries, keyed by semantic key #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub debug_info: HashMap, /// Per-pid debug info references, mapping PID to mounted modules' debug info /// Referenced by `path_keys` that point to the deduplicated `debug_info` entries. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_debug_info_by_pid: HashMap>, /// Per-pid unwind data references, mapping PID to mounted modules' unwind data /// Referenced by `path_keys` that point to the deduplicated `unwind_data` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_unwind_data_by_pid: HashMap>, /// Per-pid symbol references, mapping PID to its mounted modules' symbols /// Referenced by `path_keys` that point to the deduplicated `symbols.map` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_module_symbols: HashMap>, /// Mapping from semantic `path_key` to original binary path on host disk @@ -49,6 +57,22 @@ pub struct WalltimeMetadata { /// Until now, only kept for traceability, if we ever need to reconstruct the original paths from the keys #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub path_key_to_path: HashMap, +} + +#[derive(Serialize, Deserialize, Default)] +pub struct WalltimeMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub ignored_modules_by_pid: HashMap>, + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, // Deprecated fields below are kept for backward compatibility, since this struct is used in // the parser and older versions of the runner still generate them @@ -85,3 +109,142 @@ impl WalltimeMetadata { Ok(()) } } + +/// Companion to the memtrack event stream: the modules its allocation stacks +/// resolve against. Memory mode records benchmark boundaries in +/// `ExecutionTimestamps`, so unlike [`WalltimeMetadata`] it carries no markers. +#[derive(Serialize, Deserialize, Default)] +pub struct MemtrackMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, +} + +impl MemtrackMetadata { + pub fn from_reader(reader: R) -> anyhow::Result { + serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") + } + + pub fn save_to>(&self, path: P) -> anyhow::Result<()> { + let file = std::fs::File::create(path.as_ref().join("memtrack.metadata"))?; + const BUFFER_SIZE: usize = 256 * 1024 /* 256 KB */; + + let writer = BufWriter::with_capacity(BUFFER_SIZE, file); + serde_json::to_writer(writer, self)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Captured from the flat `WalltimeMetadata` that predates + /// [`ModuleArtifacts`]: flattening must not move a single byte, since the + /// parser reads this format from runners of every version. + const WALLTIME_JSON: &str = r#"{"version":7,"integration":["codspeed-rust","4.2.0"],"ignored_modules_by_pid":{"42":[["/lib/libpython.so",4096,8192]]},"debug_info":{"0__libc.so.6":{"object_path":"/lib/libc.so.6","addr_bounds":[4096,36864],"load_bias":4096,"debug_infos":[{"addr":4352,"size":32,"name":"malloc","file":"malloc.c","line":11}]}},"mapped_process_debug_info_by_pid":{"42":[{"debug_info_key":"0__libc.so.6","load_bias":4096}]},"mapped_process_unwind_data_by_pid":{"42":[{"unwind_data_key":"0__libc.so.6","timestamp":1234,"avma_range":{"start":4096,"end":36864},"base_avma":4096}]},"mapped_process_module_symbols":{"42":[{"perf_map_key":"0__libc.so.6","load_bias":4096}]},"path_key_to_path":{"0__libc.so.6":"/lib/libc.so.6"},"uri_by_ts":[[1,"bench::a"]],"ignored_modules":[],"markers":[]}"#; + + fn populated_artifacts() -> ModuleArtifacts { + ModuleArtifacts { + debug_info: HashMap::from([( + "0__libc.so.6".to_string(), + ModuleDebugInfo { + object_path: "/lib/libc.so.6".to_string(), + addr_bounds: (0x1000, 0x9000), + load_bias: 0x1000, + debug_infos: vec![crate::debug_info::DebugInfo { + addr: 0x1100, + size: 0x20, + name: "malloc".to_string(), + file: "malloc.c".to_string(), + line: Some(11), + }], + }, + )]), + mapped_process_debug_info_by_pid: HashMap::from([( + 42, + vec![MappedProcessDebugInfo { + debug_info_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + mapped_process_unwind_data_by_pid: HashMap::from([( + 42, + vec![MappedProcessUnwindData { + unwind_data_key: "0__libc.so.6".to_string(), + inner: crate::unwind_data::ProcessUnwindData { + timestamp: Some(1234), + avma_range: 0x1000..0x9000, + base_avma: 0x1000, + }, + }], + )]), + mapped_process_module_symbols: HashMap::from([( + 42, + vec![crate::module_symbols::MappedProcessModuleSymbols { + perf_map_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + path_key_to_path: HashMap::from([( + "0__libc.so.6".to_string(), + PathBuf::from("/lib/libc.so.6"), + )]), + } + } + + #[test] + fn walltime_metadata_serialization_is_unchanged_by_flattening() { + #[allow(deprecated)] + let metadata = WalltimeMetadata { + version: 7, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + ignored_modules_by_pid: HashMap::from([( + 42, + vec![("/lib/libpython.so".to_string(), 0x1000, 0x2000)], + )]), + artifacts: populated_artifacts(), + uri_by_ts: vec![(1, "bench::a".to_string())], + ignored_modules: vec![], + markers: vec![], + debug_info_by_pid: HashMap::new(), + }; + + assert_eq!(serde_json::to_string(&metadata).unwrap(), WALLTIME_JSON); + } + + #[test] + fn walltime_metadata_round_trips_through_the_flattened_fields() { + let parsed = WalltimeMetadata::from_reader(WALLTIME_JSON.as_bytes()).unwrap(); + + assert_eq!(parsed.artifacts.path_key_to_path.len(), 1); + assert_eq!( + parsed.artifacts.mapped_process_unwind_data_by_pid[&42].len(), + 1 + ); + assert_eq!(serde_json::to_string(&parsed).unwrap(), WALLTIME_JSON); + } + + #[test] + fn memtrack_metadata_round_trips() { + let metadata = MemtrackMetadata { + version: 1, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + artifacts: populated_artifacts(), + }; + + let json = serde_json::to_string(&metadata).unwrap(); + let parsed = MemtrackMetadata::from_reader(json.as_bytes()).unwrap(); + + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + assert_eq!( + parsed.artifacts.mapped_process_module_symbols[&42][0].perf_map_key, + "0__libc.so.6" + ); + } +} diff --git a/crates/runner-shared/src/serde_pid_map.rs b/crates/runner-shared/src/serde_pid_map.rs new file mode 100644 index 000000000..fa6fbb9dc --- /dev/null +++ b/crates/runner-shared/src/serde_pid_map.rs @@ -0,0 +1,36 @@ +//! `#[serde(with = ...)]` support for pid-keyed maps. +//! +//! JSON object keys are always strings. serde_json's direct deserializer +//! special-cases that and parses integer map keys, but a `#[serde(flatten)]` +//! field is buffered into serde's internal `Content` first, and that path has no +//! such special case — an `i32` key then fails with `invalid type: string`. So +//! the keys are read as strings and parsed here, which works on both paths. + +use libc::pid_t; +use serde::de::{Deserializer, Error}; +use serde::{Deserialize, Serialize, Serializer}; +use std::collections::HashMap; + +pub fn serialize(map: &HashMap, serializer: S) -> Result +where + V: Serialize, + S: Serializer, +{ + map.serialize(serializer) +} + +pub fn deserialize<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} diff --git a/src/executor/shared/module_artifacts/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs index 9b4899427..3e8903ed9 100644 --- a/src/executor/shared/module_artifacts/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -6,18 +6,17 @@ use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; use runner_shared::debug_info::{MappedProcessDebugInfo, ModuleDebugInfo}; +use runner_shared::metadata::ModuleArtifacts; use runner_shared::module_symbols::MappedProcessModuleSymbols; use runner_shared::unwind_data::{MappedProcessUnwindData, ProcessUnwindData, UnwindData}; use std::collections::HashMap; use std::path::{Path, PathBuf}; pub struct SavedArtifacts { - pub symbol_pid_mappings_by_pid: HashMap>, - pub debug_info: HashMap, - pub mapped_process_debug_info_by_pid: HashMap>, - pub mapped_process_unwind_data_by_pid: HashMap>, + pub artifacts: ModuleArtifacts, + /// Kept out of [`ModuleArtifacts`] because only the folded walltime trace + /// drops modules; other modes carry every module they mapped. pub ignored_modules_by_pid: HashMap>, - pub key_to_path: HashMap, } /// Save all artifacts (symbols, debug info, unwind data) from mounted modules and JIT data. @@ -30,7 +29,7 @@ pub fn save_artifacts( register_paths(&mut path_to_key, loaded_modules_by_path); - let symbol_pid_mappings_by_pid = + let mapped_process_module_symbols = save_symbols(profile_folder, loaded_modules_by_path, &path_to_key); let (debug_info, mapped_process_debug_info_by_pid) = @@ -45,18 +44,20 @@ pub fn save_artifacts( let ignored_modules_by_pid = collect_ignored_modules(loaded_modules_by_path); - let key_to_path = path_to_key + let path_key_to_path = path_to_key .into_iter() .map(|(path, key)| (key, path)) .collect(); SavedArtifacts { - symbol_pid_mappings_by_pid, - debug_info, - mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid, + artifacts: ModuleArtifacts { + debug_info, + mapped_process_debug_info_by_pid, + mapped_process_unwind_data_by_pid, + mapped_process_module_symbols, + path_key_to_path, + }, ignored_modules_by_pid, - key_to_path, } } diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 625407e86..7f31921e2 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -300,11 +300,7 @@ impl BenchmarkData<'_> { uri_by_ts: self.marker_result.uri_by_ts.clone(), ignored_modules_by_pid: artifacts.ignored_modules_by_pid, markers: self.marker_result.markers.clone(), - debug_info: artifacts.debug_info, - mapped_process_debug_info_by_pid: artifacts.mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid: artifacts.mapped_process_unwind_data_by_pid, - mapped_process_module_symbols: artifacts.symbol_pid_mappings_by_pid, - path_key_to_path: artifacts.key_to_path, + artifacts: artifacts.artifacts, // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), ignored_modules: Default::default(), diff --git a/src/executor/wall_time/profiler/samply/mod.rs b/src/executor/wall_time/profiler/samply/mod.rs index 3d77e7ade..5f04ef8c8 100644 --- a/src/executor/wall_time/profiler/samply/mod.rs +++ b/src/executor/wall_time/profiler/samply/mod.rs @@ -184,11 +184,7 @@ impl Profiler for SamplyProfiler { // These fields aren't required in samply, since we symbolicate client-side. ignored_modules_by_pid: Default::default(), - debug_info: Default::default(), - mapped_process_debug_info_by_pid: Default::default(), - mapped_process_unwind_data_by_pid: Default::default(), - mapped_process_module_symbols: Default::default(), - path_key_to_path: Default::default(), + artifacts: Default::default(), // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), From 232b7a8fd171066e0f625bb11c44f88fc20bf309 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:13:30 +0200 Subject: [PATCH 07/18] feat(memtrack): record mapped modules for offline stack attribution Allocation stacks are raw addresses, so resolving them off-box needs the module geometry perf gets from PERF_RECORD_MMAP2. No single hook provides it: security_mmap_file has the file but runs before the VMA exists, and perf_event_mmap has the addresses but cannot resolve a path. So an LSM program caches the path once per inode and a perf_event_mmap fentry emits inode-keyed address records, joined in userspace while the maps are still live. Path resolution is only reachable from an LSM program at all, and only above 5.11 (bpf_d_path on the sleepable hook) or 6.12 (the bpf_path_d_path kfunc), with the bpf LSM active. MappingSupport probes both, and when neither holds stack capture is turned off rather than shipping stacks nothing can attribute. --- crates/memtrack/src/ebpf/c/attach.h | 5 - crates/memtrack/src/ebpf/c/event.h | 21 +++ crates/memtrack/src/ebpf/c/main.bpf.c | 1 + crates/memtrack/src/ebpf/c/mappings.bpf.h | 161 ++++++++++++++++++ crates/memtrack/src/ebpf/mappings/mod.rs | 7 + crates/memtrack/src/ebpf/mappings/records.rs | 83 +++++++++ crates/memtrack/src/ebpf/mappings/resolve.rs | 73 ++++++++ crates/memtrack/src/ebpf/mappings/support.rs | 103 +++++++++++ crates/memtrack/src/ebpf/memtrack/maps.rs | 52 ++++++ crates/memtrack/src/ebpf/memtrack/mod.rs | 63 ++++++- crates/memtrack/src/ebpf/memtrack/tracking.rs | 23 +++ crates/memtrack/src/ebpf/mod.rs | 3 +- crates/memtrack/src/ebpf/tracker.rs | 80 +++++++-- crates/memtrack/src/session.rs | 3 + crates/memtrack/tests/shared.rs | 6 +- crates/memtrack/tests/stack_tests.rs | 47 +---- .../src/artifacts/memtrack/mappings.rs | 37 ++++ .../src/artifacts/memtrack/mod.rs | 2 + 18 files changed, 695 insertions(+), 75 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/mappings.bpf.h create mode 100644 crates/memtrack/src/ebpf/mappings/mod.rs create mode 100644 crates/memtrack/src/ebpf/mappings/records.rs create mode 100644 crates/memtrack/src/ebpf/mappings/resolve.rs create mode 100644 crates/memtrack/src/ebpf/mappings/support.rs create mode 100644 crates/runner-shared/src/artifacts/memtrack/mappings.rs diff --git a/crates/memtrack/src/ebpf/c/attach.h b/crates/memtrack/src/ebpf/c/attach.h index e188c7d5f..90cbe4360 100644 --- a/crates/memtrack/src/ebpf/c/attach.h +++ b/crates/memtrack/src/ebpf/c/attach.h @@ -14,11 +14,6 @@ #define MEMTRACK_PROT_EXEC 0x4 #define MEMTRACK_SIGSTOP 19 -struct inode_key { - __u64 dev; - __u64 ino; -}; - /* (dev, ino) -> 1; populated by userspace after classify/attach */ BPF_HASH_MAP(known_inodes, struct inode_key, __u8, 8192); /* Requests are 24 B and rare; overflow aborts the run via the counter below */ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 7e10a7756..e3fdab0aa 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -115,6 +115,13 @@ struct event { } data; }; +/* Identifies a mapped file across both the attach watcher and the mapping + * recorder. `dev` uses the kernel's s_dev encoding: (major << 20) | minor. */ +struct inode_key { + uint64_t dev; + uint64_t ino; +}; + /* Request from the exec-mapping watcher to the userspace attach worker */ struct attach_request { uint32_t pid; @@ -122,4 +129,18 @@ struct attach_request { uint64_t ino; }; +/* One executable file mapping, mirroring PERF_RECORD_MMAP2. The path is not + * here: it is resolved once per inode into a BPF map that userspace joins + * against, since every mapping of the same file shares it. */ +struct mapping_record { + uint64_t dev; + uint64_t ino; + uint64_t file_offset; /* offset of the mapping's first byte in the file */ + uint64_t start; + uint64_t end; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + uint32_t pid; + uint32_t _pad; +}; + #endif /* __EVENT_H__ */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index b405f572b..7a068c605 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,6 +8,7 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "mappings.bpf.h" #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h new file mode 100644 index 000000000..5b5635a12 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/mappings.bpf.h @@ -0,0 +1,161 @@ +#ifndef __MAPPINGS_BPF_H__ +#define __MAPPINGS_BPF_H__ + +#include "event.h" +#include "utils/folio.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* == Mapping recorder == + * + * Reconstructs what `PERF_RECORD_MMAP2` gives perf: which file a tracked + * process mapped, where, so raw stack addresses can be attributed to modules + * offline. No single hook carries both halves: + * + * security_mmap_file(file, ..) has the file, runs before the VMA exists + * perf_event_mmap(vma) has the addresses, cannot resolve a path + * + * The path therefore lands in a per-inode cache, and the address-bearing hook + * emits inode-keyed records that userspace joins against that cache while this + * BPF object is still loaded. + * + * Path resolution is only reachable from an LSM program: `bpf_d_path()` is + * restricted to sleepable LSM hooks, `BPF_TRACE_ITER` and an fentry allowlist + * holding no mmap path, and the newer `bpf_path_d_path()` kfunc rejects + * non-LSM program types. Both variants are compiled; userspace autoloads the + * one the running kernel supports and neither when the bpf LSM is inactive. */ + +/* VM_EXEC from linux/mm.h, which vmlinux.h does not carry (it is a macro, not a + * type). Only executable mappings are recorded: unwind data and symbols are + * looked up by text address. */ +#define MEMTRACK_VM_EXEC 0x00000004 + +/* d_path() fails with -ENAMETOOLONG rather than truncating, so a short buffer + * loses whole modules. PATH_MAX keeps that from happening. */ +#define MEMTRACK_MAX_PATH 4096 + +struct inode_path { + __u32 len; /* bytes written by d_path, including the NUL */ + char path[MEMTRACK_MAX_PATH]; +}; + +/* Every mapping of an inode shares its cached path. */ +BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); + +/* A dropped record may leave a module unresolved. */ +BPF_RINGBUF(mappings, 256 * 1024); +BPF_ARRAY_MAP(mapping_dropped, __u64, 1); + +/* The path does not fit on the BPF stack; build it in this per-CPU scratch map. */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct inode_path); +} path_scratch SEC(".maps"); + +extern int bpf_path_d_path(const struct path* path, char* buf, __u64 buf__sz) __ksym __weak; + +static __always_inline void bump_mapping_dropped(void) { + __u32 zero = 0; + __u64* drops = bpf_map_lookup_elem(&mapping_dropped, &zero); + if (drops) { + __sync_fetch_and_add(drops, 1); + } +} + +/* Return a scratch slot when this inode has no cached path. */ +static __always_inline struct inode_path* mapping_path_slot(struct file* file, + struct inode_key* key) { + if (!file || !is_tracked(current_tgid())) { + return NULL; + } + + key->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + key->ino = BPF_CORE_READ(file, f_inode, i_ino); + if (bpf_map_lookup_elem(&path_by_inode, key)) { + return NULL; + } + + __u32 zero = 0; + return bpf_map_lookup_elem(&path_scratch, &zero); +} + +/* Publish a resolved path. A failed resolution is not cached, so the next + * mapping of the same inode retries instead of losing the module for the run. */ +static __always_inline void commit_mapping_path(struct inode_key* key, struct inode_path* entry, + int len) { + if (len <= 0) { + return; + } + entry->len = (__u32)len; + bpf_map_update_elem(&path_by_inode, key, entry, BPF_NOEXIST); +} + +/* Kernels >= 6.12: the kfunc is callable from any LSM program. */ +SEC("lsm/mmap_file") +int BPF_PROG(cache_mmap_path_kfunc, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, + bpf_path_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* Kernels 5.11..6.11: `bpf_d_path()` needs a sleepable LSM hook, which + * `mmap_file` has been since 5.11. */ +SEC("lsm.s/mmap_file") +int BPF_PROG(cache_mmap_path_legacy, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, bpf_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* The same hook perf emits MMAP2 from, so the recorded geometry matches what + * the walltime pipeline already consumes: the file offset is in bytes, not + * pages. */ +SEC("fentry/perf_event_mmap") +int BPF_PROG(record_mmap, struct vm_area_struct* vma) { + if (!vma) { + return 0; + } + + __u32 tgid = current_tgid(); + if (!is_tracked(tgid)) { + return 0; + } + + struct file* file = BPF_CORE_READ(vma, vm_file); + if (!file) { + return 0; + } + if (!(BPF_CORE_READ(vma, vm_flags) & MEMTRACK_VM_EXEC)) { + return 0; + } + + struct mapping_record* rec = bpf_ringbuf_reserve(&mappings, sizeof(*rec), 0); + if (!rec) { + bump_mapping_dropped(); + return 0; + } + + rec->pid = tgid; + rec->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + rec->ino = BPF_CORE_READ(file, f_inode, i_ino); + rec->file_offset = (__u64)BPF_CORE_READ(vma, vm_pgoff) << page_shift; + rec->start = BPF_CORE_READ(vma, vm_start); + rec->end = BPF_CORE_READ(vma, vm_end); + rec->timestamp = bpf_ktime_get_ns(); + bpf_ringbuf_submit(rec, 0); + + return 0; +} + +#endif /* __MAPPINGS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/mappings/mod.rs b/crates/memtrack/src/ebpf/mappings/mod.rs new file mode 100644 index 000000000..581de916b --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/mod.rs @@ -0,0 +1,7 @@ +mod records; +mod resolve; +mod support; + +pub(crate) use records::MappingRecord; +pub(crate) use resolve::resolve_mappings; +pub use support::MappingSupport; diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs new file mode 100644 index 000000000..4c660031e --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/records.rs @@ -0,0 +1,83 @@ +use crate::ebpf::events::bindings::mapping_record; + +/// One executable file mapping as the BPF recorder saw it. The path is resolved +/// separately, per inode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MappingRecord { + pub pid: u32, + pub dev: u64, + pub ino: u64, + pub file_offset: u64, + pub start: u64, + pub end: u64, + pub timestamp: u64, +} + +impl MappingRecord { + pub fn parse(data: &[u8]) -> Option { + if data.len() < std::mem::size_of::() { + return None; + } + + // SAFETY: the length is checked above, and the layout is the + // bindgen-generated C ABI struct. + let record: mapping_record = unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; + Some(Self { + pid: record.pid, + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + start: record.start, + end: record.end, + timestamp: record.timestamp, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode(record: mapping_record) -> Vec { + // SAFETY: reading a plain-data struct as bytes. + unsafe { + std::slice::from_raw_parts( + (&record as *const mapping_record).cast::(), + std::mem::size_of::(), + ) + } + .to_vec() + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let bytes = encode(mapping_record { + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + pid: 7, + _pad: 0, + }); + + assert_eq!( + MappingRecord::parse(&bytes), + Some(MappingRecord { + pid: 7, + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + }) + ); + } + + #[test] + fn truncated_buffer_returns_none() { + assert!(MappingRecord::parse(&[0u8; 8]).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs new file mode 100644 index 000000000..9c20ba7f1 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/resolve.rs @@ -0,0 +1,73 @@ +use super::MappingRecord; +use crate::prelude::*; +use runner_shared::artifacts::ProcessMapping; +use std::collections::HashMap; + +/// Records without a path are dropped because their unwind data and symbols cannot be read. +pub(crate) fn resolve_mappings( + records: Vec, + paths: &HashMap<(u64, u64), String>, +) -> Vec { + let mut unresolved = 0; + let mappings = records + .into_iter() + .filter_map(|record| { + let Some(path) = paths.get(&(record.dev, record.ino)) else { + unresolved += 1; + return None; + }; + + Some(ProcessMapping { + pid: record.pid as i32, + path: path.clone(), + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + avma_range: record.start..record.end, + timestamp: record.timestamp, + }) + }) + .collect(); + + if unresolved > 0 { + debug!("{unresolved} mapping records had no resolved path and were dropped"); + } + mappings +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(dev: u64, ino: u64) -> MappingRecord { + MappingRecord { + pid: 5, + dev, + ino, + file_offset: 0x1000, + start: 0x4000, + end: 0x8000, + timestamp: 42, + } + } + + #[test] + fn resolves_records_against_the_path_cache() { + let paths = HashMap::from([((1, 2), "/lib/libc.so.6".to_string())]); + + let mappings = resolve_mappings(vec![record(1, 2)], &paths); + + assert_eq!(mappings.len(), 1); + assert_eq!(mappings[0].path, "/lib/libc.so.6"); + assert_eq!(mappings[0].avma_range, 0x4000..0x8000); + assert_eq!(mappings[0].file_offset, 0x1000); + assert_eq!(mappings[0].pid, 5); + } + + /// A module we cannot name is a module we cannot read, so it must not reach + /// the artifact as an empty path. + #[test] + fn drops_records_without_a_resolved_path() { + assert!(resolve_mappings(vec![record(9, 9)], &HashMap::new()).is_empty()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/support.rs b/crates/memtrack/src/ebpf/mappings/support.rs new file mode 100644 index 000000000..9e2f3a190 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/support.rs @@ -0,0 +1,103 @@ +use crate::kernel::KernelVersion; +use crate::prelude::*; + +/// How the running kernel can resolve a mapped file's path inside BPF. +/// +/// Only a BPF LSM program can do it at all: `bpf_d_path()` is restricted to +/// `BPF_TRACE_ITER` programs, sleepable LSM hooks and a fixed fentry allowlist +/// that contains no mmap path (`bpf_d_path_allowed()` in +/// `kernel/trace/bpf_trace.c`), and the `bpf_path_d_path()` kfunc that replaces +/// it rejects every program type but LSM (`bpf_fs_kfuncs_filter()` in +/// `fs/bpf_fs_kfuncs.c`). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum MappingSupport { + /// Paths cannot be resolved, so allocation stacks could not be attributed to + /// modules and are not worth capturing. + Unsupported, + /// Sleepable LSM hook calling `bpf_d_path()` (kernel >= 5.11). + Legacy, + /// LSM hook calling the `bpf_path_d_path()` kfunc (kernel >= 6.12). + Kfunc, +} + +impl MappingSupport { + /// What the running kernel and its boot configuration provide. + /// + /// The kernel release is only half the gate: `bpf` must also be in the + /// active LSM list, which is fixed at boot by `CONFIG_LSM`/`lsm=` and cannot + /// be inferred from the version. + pub fn detect() -> Self { + if !bpf_lsm_active() { + info!( + "The bpf LSM is not active (see /sys/kernel/security/lsm), so mapped module paths \ + cannot be resolved" + ); + return Self::Unsupported; + } + + let version = match KernelVersion::current() { + Ok(version) => version, + Err(e) => { + warn!("Failed to read the kernel version, no mapping records: {e:#}"); + return Self::Unsupported; + } + }; + + let support = Self::for_version(version); + match support { + Self::Unsupported => { + info!("Kernel {version} cannot resolve paths from an LSM program (needs >= 5.11)") + } + Self::Legacy => { + debug!("Kernel {version} predates the bpf_path_d_path kfunc, using bpf_d_path") + } + Self::Kfunc => {} + } + support + } + + fn for_version(version: KernelVersion) -> Self { + if version < KernelVersion::new(5, 11) { + return Self::Unsupported; + } + if version < KernelVersion::new(6, 12) { + return Self::Legacy; + } + Self::Kfunc + } +} + +/// Whether `bpf` is one of the LSMs the running kernel initialized. An +/// unreadable file means securityfs is not mounted, in which case no LSM program +/// will attach either. +fn bpf_lsm_active() -> bool { + const PATH: &str = "/sys/kernel/security/lsm"; + + let Ok(active) = std::fs::read_to_string(PATH) else { + debug!("Could not read {PATH} to check whether the bpf LSM is active"); + return false; + }; + active.trim().split(',').any(|lsm| lsm == "bpf") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `bpf_lsm_mmap_file` has been in the sleepable hook set since 5.11, and + /// 6.12 is the first release carrying `bpf_path_d_path`. + #[test] + fn maps_releases_to_support_levels() { + for (major, minor, expected) in [ + (5, 4, MappingSupport::Unsupported), + (5, 10, MappingSupport::Unsupported), + (5, 11, MappingSupport::Legacy), + (6, 11, MappingSupport::Legacy), + (6, 12, MappingSupport::Kfunc), + (7, 1, MappingSupport::Kfunc), + ] { + let version = KernelVersion::new(major, minor); + assert_eq!(MappingSupport::for_version(version), expected, "{version}"); + } + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index 994650d53..fa8f6f920 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -2,6 +2,7 @@ use super::MemtrackBpf; use crate::ebpf::stacks::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; +use std::collections::HashMap; impl MemtrackBpf { pub fn add_tracked_pid(&mut self, pid: i32) -> Result<()> { @@ -62,6 +63,39 @@ impl MemtrackBpf { ) } + /// Number of mapping records dropped because their ring buffer was full. + /// A non-zero value means a module may be missing from the trace. + pub fn mapping_dropped_count(&self) -> Result { + read_counter( + with_skel!(self, skel => &skel.maps.mapping_dropped), + "mapping_dropped", + ) + } + + /// The paths the kernel resolved for every mapped file, keyed by + /// `(dev, ino)`. Only readable while the BPF object is alive. + pub fn mapped_paths(&self) -> Result> { + let map = with_skel!(self, skel => &skel.maps.path_by_inode); + + let mut paths = HashMap::new(); + for key in map.keys() { + let Some(value) = map + .lookup(&key, libbpf_rs::MapFlags::ANY) + .context("Failed to read a resolved mapping path")? + else { + continue; + }; + + let Some((dev, ino)) = inode_key(&key) else { + continue; + }; + if let Some(path) = inode_path(&value) { + paths.insert((dev, ino), path); + } + } + Ok(paths) + } + pub fn dropped_events_count(&self) -> Result { read_counter( with_skel!(self, skel => &skel.maps.dropped_events), @@ -118,6 +152,24 @@ fn le(bytes: &[u8]) -> u64 { .fold(0, |acc, &b| acc << 8 | u64::from(b)) } +/// Split a `struct inode_key { __u64 dev; __u64 ino; }` map key. +fn inode_key(key: &[u8]) -> Option<(u64, u64)> { + if key.len() < 16 { + return None; + } + Some((le(&key[..8]), le(&key[8..16]))) +} + +/// Read a `struct inode_path { __u32 len; char path[]; }` map value. The kernel +/// wrote `len` bytes including the NUL terminator. +fn inode_path(value: &[u8]) -> Option { + const PATH_OFFSET: usize = 4; + + let len = u32::from_le_bytes(value.get(..PATH_OFFSET)?.try_into().ok()?) as usize; + let path = value.get(PATH_OFFSET..PATH_OFFSET + len.saturating_sub(1))?; + Some(String::from_utf8_lossy(path).into_owned()) +} + /// Read slot 0 of a single-entry `__u64` array map. fn read_counter(map: &impl MapCore, name: &str) -> Result { let key = 0u32; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 44705cc25..c3670c6ea 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; +use crate::ebpf::mappings::MappingSupport; use crate::ebpf::poller::RingBufferPoller; mod token { @@ -119,28 +120,35 @@ pub struct MemtrackBpf { pub(super) skel: Skel, pub(super) probes: Vec, rmap: RmapSupport, + pub(super) mappings: MappingSupport, } impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool, stack_copy_size: Option) -> Result { + pub fn new_with_rmap( + track_rmap: bool, + capture_stacks: bool, + mappings: MappingSupport, + ) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap, stack_copy_size) + Self::with_variant(variant, track_rmap, capture_stacks, mappings) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. /// - /// `stack_copy_size` turns on allocation stack capture. + /// `capture_stacks` enables allocation stack capture, and `mappings` + /// selects the path-resolving LSM program the running kernel supports. pub fn with_variant( variant: BpfVariant, track_rmap: bool, - stack_copy_size: Option, + capture_stacks: bool, + mappings: MappingSupport, ) -> Result { crate::kernel::KernelBtf::ensure_available()?; @@ -172,15 +180,14 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } - if let Some(copy_size) = stack_copy_size { + if capture_stacks { rodata.capture_stacks_enabled = 1; - rodata.stack_copy_size = copy_size; } } // Avoid reserving the stack maps when capture is disabled. A // ring buffer's size must stay a power-of-two page count. - if stack_copy_size.is_none() { + if !capture_stacks { open_skel.maps.stacks.set_max_entries(4096)?; open_skel.maps.stack_traces.set_max_entries(1)?; open_skel.maps.seen_stack_hashes.set_max_entries(1)?; @@ -209,6 +216,26 @@ impl MemtrackBpf { RmapSupport::CoreAndPud => {} } + // The kfunc variant fails to load on kernels without + // `bpf_path_d_path`, and neither LSM program can attach when the + // bpf LSM is inactive; without a path there is nothing to + // resolve records against, so the recorder goes too. + match mappings { + MappingSupport::Unsupported => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + open_skel.progs.record_mmap.set_autoload(false); + open_skel.maps.mappings.set_max_entries(4096)?; + open_skel.maps.path_by_inode.set_max_entries(1)?; + } + MappingSupport::Legacy => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + } + MappingSupport::Kfunc => { + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + } + } + $skel(Box::new( open_skel .load() @@ -230,6 +257,7 @@ impl MemtrackBpf { skel, probes: Vec::new(), rmap, + mappings, }) } @@ -295,6 +323,27 @@ impl MemtrackBpf { )) } + /// Poll the mapping-record ring buffer into `tx`. Same contract as + /// [`Self::poll_events_with_channel`]. + pub(crate) fn poll_mappings_with_channel( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.mappings, + crate::ebpf::mappings::MappingRecord::parse, + tx, + poll_interval_ms, + )) + } + + /// Whether the mapping recorder is loaded, i.e. whether its ring buffer is + /// worth polling. + pub fn records_mappings(&self) -> bool { + self.mappings != MappingSupport::Unsupported + } + /// Number of currently-attached probes/links. pub fn probe_count(&self) -> usize { self.probes.len() diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 3e00ceb3b..350c039ee 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -1,4 +1,5 @@ use super::{MemtrackBpf, RmapSupport}; +use crate::ebpf::mappings::MappingSupport; use crate::prelude::*; use paste::paste; @@ -64,4 +65,26 @@ impl MemtrackBpf { self.probes.push(link); Ok(()) } + + /// Attach the mapping recorder: the LSM hook caching resolved paths and the + /// `perf_event_mmap` fentry emitting the address records. Only the LSM + /// variant the running kernel supports was loaded. + pub fn attach_mapping_recorder(&mut self) -> Result<()> { + let link = match self.mappings { + MappingSupport::Unsupported => return Ok(()), + MappingSupport::Legacy => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_legacy.attach()) + } + MappingSupport::Kfunc => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_kfunc.attach()) + } + } + .context("Failed to attach the mmap path resolver")?; + self.probes.push(link); + + let link = with_skel!(mut self, skel => skel.progs.record_mmap.attach()) + .context("Failed to attach the mapping recorder")?; + self.probes.push(link); + Ok(()) + } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index d964ebed5..743970c86 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -1,5 +1,6 @@ mod attach_worker; mod events; +pub(crate) mod mappings; mod memtrack; pub(crate) mod poller; mod proc_fs; @@ -7,9 +8,9 @@ mod spawn; mod stacks; mod tracker; +pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; -pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; pub use stacks::counters::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 5536e5324..13ed9145f 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,11 +1,13 @@ use crate::ebpf::attach_worker::AttachWorker; +use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::stacks::config::{clamp_copy_size, stack_copy_size_from_env}; +use crate::ebpf::stacks::config::stack_capture_from_env; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; +use runner_shared::artifacts::MemtrackMappings; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; @@ -22,10 +24,9 @@ pub struct TrackerOptions { /// Reconstruct per-process RSS from the folio rmap fentry hooks. #[builder(default = false)] pub rmap: bool, - /// Bytes of user stack to copy for each allocation event. `None` leaves - /// stack capture off; values are clamped to the supported range. - #[builder(default = None)] - pub stack_copy_size: Option, + /// Capture allocation call stacks. + #[builder(default = true)] + pub stack_capture: bool, } impl TrackerOptions { @@ -36,7 +37,7 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .rmap(std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1")) - .stack_copy_size(stack_copy_size_from_env()) + .stack_capture(stack_capture_from_env()) .build() } } @@ -48,6 +49,9 @@ pub struct Tracker { /// The dedup gate spans the whole BPF object, so a second session would /// reference stack records the first one already consumed. stacks_polled: Option, + /// Filled by the mapping poller; drained by [`Tracker::mappings`] after the + /// session is dropped, so the poller's final drain is included. + mapping_rx: Mutex>>, } impl Tracker { @@ -59,20 +63,29 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { - let copy_size = options.stack_copy_size.map(clamp_copy_size); + let mappings = MappingSupport::detect(); + + // Stacks are raw addresses: without mapped module paths nothing can + // attribute them, so capturing them would only inflate the artifact. + let capture_stacks = match (options.stack_capture, mappings) { + (true, MappingSupport::Unsupported) => { + warn!("Allocation stack capture needs in-kernel path resolution; disabling it"); + false + } + (capture_stacks, _) => capture_stacks, + }; Self::build( - MemtrackBpf::new_with_rmap(options.rmap, copy_size)?, + MemtrackBpf::new_with_rmap(options.rmap, capture_stacks, mappings)?, options.allocators, - copy_size.is_some(), + capture_stacks, ) } - /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = TrackerOptions::from_env().rmap; Self::build( - MemtrackBpf::with_variant(variant, track_rmap, None)?, + MemtrackBpf::with_variant(variant, track_rmap, false, MappingSupport::detect())?, true, false, ) @@ -87,6 +100,7 @@ impl Tracker { bpf.attach_tracepoints()?; if allocators { bpf.attach_exec_watcher()?; + bpf.attach_mapping_recorder()?; } let bpf = Arc::new(Mutex::new(bpf)); @@ -101,6 +115,7 @@ impl Tracker { worker: Mutex::new(worker), allocators, stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), + mapping_rx: Mutex::new(None), }) } @@ -137,17 +152,54 @@ impl Tracker { } let (tx, rx) = mpsc::channel(); - let (poller, stack_poller) = { + let (mapping_tx, mapping_rx) = mpsc::channel(); + let (poller, stack_poller, mapping_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; let stack_poller = capture_stacks .then(|| bpf.poll_stacks(10, tx.clone())) .transpose()?; - (bpf.poll_events_with_channel(10, tx)?, stack_poller) + let mapping_poller = bpf + .records_mappings() + .then(|| bpf.poll_mappings_with_channel(10, mapping_tx)) + .transpose()?; + ( + bpf.poll_events_with_channel(10, tx)?, + stack_poller, + mapping_poller, + ) }; + *self.mapping_rx.lock() = Some(mapping_rx); resume(pid)?; - Ok(Session::new(child, rx, poller, stack_poller)) + Ok(Session::new( + child, + rx, + poller, + stack_poller, + mapping_poller, + )) + } + + /// The module mappings recorded during the run, joined with the paths the + /// kernel resolved for them. Call after dropping the session so the poller's + /// final drain is included, and before the BPF object is torn down. + pub fn mappings(&self) -> Result { + let Some(rx) = self.mapping_rx.lock().take() else { + return Ok(MemtrackMappings::default()); + }; + + let records: Vec<_> = rx.try_iter().collect(); + let paths = self.bpf.lock().mapped_paths()?; + + let dropped = self.bpf.lock().mapping_dropped_count()?; + if dropped > 0 { + warn!("{dropped} mapping records were dropped; some modules may be unresolved"); + } + + Ok(MemtrackMappings { + mappings: resolve_mappings(records, &paths), + }) } /// Enable allocator-event tracking in the BPF program. Lifetime events diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 9bed66b7d..551db639c 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -11,6 +11,7 @@ pub struct Session { events: Option>, _poller: RingBufferPoller, _stack_poller: Option, + _mapping_poller: Option, } impl Session { @@ -19,12 +20,14 @@ impl Session { events: Receiver, poller: RingBufferPoller, stack_poller: Option, + mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, _stack_poller: stack_poller, + _mapping_poller: mapping_poller, } } diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index b2f461b34..9bc2e2e9b 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -242,12 +242,10 @@ pub fn track_command_with_rmap_maps( } /// Track a command with allocation stack capture enabled, returning its events. -pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { +pub fn track_command_with_stacks(command: Command) -> TrackResult { track_command_with_opts( command, - TrackerOptions::builder() - .stack_copy_size(Some(copy_size)) - .build(), + TrackerOptions::builder().stack_capture(true).build(), ) } diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 7a95cd6ca..ae3e30f28 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -6,7 +6,7 @@ use std::collections::HashSet; use std::process::Command; use tempfile::TempDir; -const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; +const COPY_SIZE: u32 = 8192; fn compile_fixture( name: &str, @@ -59,8 +59,7 @@ fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box = events .iter() @@ -133,8 +132,7 @@ fn dedup_collapses_repeated_call_paths() -> Result<(), Box Result<(), Box Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_max", &temp_dir)?; - let (events, thread_handle) = - shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; - - let truncated: Vec<_> = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record: r } if r.truncated => Some(r.hash), - _ => None, - }) - .collect(); - - assert!( - !record_hashes(&events).is_empty(), - "expected stack records at the maximum copy budget" - ); - assert!( - truncated.is_empty(), - "no capture can be budget-limited at the maximum budget: {truncated:#x?}" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) -} - /// Restores the capture toggle on drop so a failing assertion cannot leak the /// override into later tests (the suite runs single-threaded). struct DisableCaptureGuard; diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs new file mode 100644 index 000000000..b271dfee4 --- /dev/null +++ b/crates/runner-shared/src/artifacts/memtrack/mappings.rs @@ -0,0 +1,37 @@ +use libc::pid_t; +use serde::{Deserialize, Serialize}; +use std::ops::Range; + +/// The file-backed mappings the tracked process tree loaded, recorded as they +/// happened. Companion to the event stream: allocation stacks are raw +/// addresses, and these are what turns them back into modules. +/// +/// Kept out of the event stream so a consumer that only needs the module set +/// does not have to decode millions of allocation events. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct MemtrackMappings { + pub mappings: Vec, +} + +impl super::super::ArtifactExt for MemtrackMappings {} + +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessMapping { + pub pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + pub path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + pub dev: u64, + pub ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + pub file_offset: u64, + pub avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + pub timestamp: u64, +} diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 433aab5a5..0ba0ced47 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -2,9 +2,11 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::io::{BufReader, Read, Write}; +mod mappings; mod pipeline; mod writer; +pub use mappings::*; pub use pipeline::*; pub use writer::*; From ebc101afb910791f129074e074d0ccbe3de022b5 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:38:28 +0200 Subject: [PATCH 08/18] feat(runner): write memtrack module artifacts and metadata Memory mode now turns the mappings memtrack recorded into the same keyed unwind_data/symbols.map files walltime writes, plus a memtrack.metadata referencing them per pid, so allocation stacks can be unwound off-box. Each mapping's inode is rechecked against the path before its ELF is read: BPF cannot produce a build id, so the recorded (dev, ino) is what proves the file on disk is still the one that was mapped rather than a rebuilt binary whose eh_frame would be bound to the wrong addresses. --- crates/runner-shared/src/metadata.rs | 12 +- src/executor/memory/executor.rs | 46 ++-- src/executor/memory/mod.rs | 1 + src/executor/memory/module_artifacts.rs | 272 ++++++++++++++++++++++++ 4 files changed, 318 insertions(+), 13 deletions(-) create mode 100644 src/executor/memory/module_artifacts.rs diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 654ae298d..2e48a2fd8 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -126,6 +126,16 @@ pub struct MemtrackMetadata { } impl MemtrackMetadata { + pub const CURRENT_VERSION: u64 = 1; + + pub fn new(integration: (String, String), artifacts: ModuleArtifacts) -> Self { + Self { + version: Self::CURRENT_VERSION, + integration, + artifacts, + } + } + pub fn from_reader(reader: R) -> anyhow::Result { serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") } @@ -233,7 +243,7 @@ mod tests { #[test] fn memtrack_metadata_round_trips() { let metadata = MemtrackMetadata { - version: 1, + version: MemtrackMetadata::CURRENT_VERSION, integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), artifacts: populated_artifacts(), }; diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8c9a3985..b8a294d8a 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback; use crate::executor::helpers::run_with_env::prefix_command_with_env; use crate::executor::helpers::run_with_sudo::is_root_user; +use crate::executor::memory::module_artifacts::save_module_artifacts; use crate::executor::memory::tunables::MemoryTunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; @@ -24,6 +25,7 @@ use runner_shared::artifacts::{ArtifactExt, ExecutionTimestamps}; use runner_shared::fifo::Command as FifoCommand; use runner_shared::fifo::IntegrationMode; use semver::Version; +use std::cell::RefCell; use std::fs::canonicalize; use std::path::Path; use std::rc::Rc; @@ -163,7 +165,8 @@ impl Executor for MemoryExecutor { let _tunables = MemoryTunables::apply(); // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions - std::fs::create_dir_all(execution_context.profile_folder.join("results"))?; + let results_folder = execution_context.profile_folder.join("results"); + std::fs::create_dir_all(&results_folder)?; Self::ensure_privileges()?; @@ -172,16 +175,18 @@ impl Executor for MemoryExecutor { debug!("cmd: {cmd:?}"); let runner_fifo = RunnerFifo::new()?; - let on_process_started = |mut child: std::process::Child| async move { - let (marker_result, exit_status) = - Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + let integration = Rc::new(RefCell::new(None)); + let on_process_started = { + let integration = integration.clone(); + |mut child: std::process::Child| async move { + let (marker_result, fifo_data, exit_status) = + Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + *integration.borrow_mut() = fifo_data.integration; - // Directly write to the profile folder, to avoid having to define another field - marker_result - .save_to(execution_context.profile_folder.join("results")) - .unwrap(); + marker_result.save_to(&results_folder).unwrap(); - Ok(exit_status) + Ok(exit_status) + } }; let status = run_command_with_log_pipe_and_callback(cmd, on_process_started).await?; @@ -191,6 +196,19 @@ impl Executor for MemoryExecutor { bail!("failed to execute memory tracker process: {status}"); } + if let Some(integration) = integration.borrow_mut().take() { + let results_folder = execution_context.profile_folder.join("results"); + if let Err(e) = save_module_artifacts( + &execution_context.profile_folder, + &results_folder, + integration, + ) { + // The memory results are complete without them; only offline + // stack attribution is lost. + error!("Failed to save memtrack module artifacts: {e:#}"); + } + } + Ok(()) } @@ -228,7 +246,11 @@ impl MemoryExecutor { mut runner_fifo: RunnerFifo, ipc: MemtrackIpcServer, child: &mut std::process::Child, - ) -> anyhow::Result<(ExecutionTimestamps, std::process::ExitStatus)> { + ) -> anyhow::Result<( + ExecutionTimestamps, + crate::executor::shared::fifo::FifoBenchmarkData, + std::process::ExitStatus, + )> { // Accept the IPC connection from memtrack and get the sender it sends us // Use a timeout to prevent hanging if the process doesn't start properly // https://github.com/servo/ipc-channel/issues/261 @@ -300,9 +322,9 @@ impl MemoryExecutor { Ok(None) }; - let (marker_result, _, exit_status) = + let (marker_result, fifo_data, exit_status) = runner_fifo.handle_fifo_messages(child, on_cmd).await?; - Ok((marker_result, exit_status)) + Ok((marker_result, fifo_data, exit_status)) } } diff --git a/src/executor/memory/mod.rs b/src/executor/memory/mod.rs index 2d17547d1..9f48a81ab 100644 --- a/src/executor/memory/mod.rs +++ b/src/executor/memory/mod.rs @@ -1,3 +1,4 @@ pub mod executor; +pub(crate) mod module_artifacts; pub(crate) mod setup; pub(crate) mod tunables; diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs new file mode 100644 index 000000000..f8dcd0e9c --- /dev/null +++ b/src/executor/memory/module_artifacts.rs @@ -0,0 +1,272 @@ +use crate::executor::shared::module_artifacts::loaded_module::LoadedModule; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; +use crate::prelude::*; +use runner_shared::artifacts::{ArtifactExt, MemtrackMappings, ProcessMapping}; +use runner_shared::metadata::MemtrackMetadata; +use std::collections::HashMap; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; + +/// Turn the mappings memtrack recorded into the artifacts an offline unwinder +/// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the +/// `memtrack.metadata` referencing them per pid. +/// +/// `results_folder` is where memtrack wrote its artifacts; the keyed files and +/// the metadata land in `profile_folder`, next to walltime's equivalents. +pub fn save_module_artifacts( + profile_folder: &Path, + results_folder: &Path, + integration: (String, String), +) -> Result<()> { + let mappings = read_mappings(results_folder)?; + if mappings.is_empty() { + debug!("No module mappings recorded, skipping memtrack module artifacts"); + return Ok(()); + } + + let loaded_modules = loaded_modules_from_mappings(&mappings); + debug!( + "Extracting artifacts for {} modules from {} mappings", + loaded_modules.len(), + mappings.len() + ); + + let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); + MemtrackMetadata { + version: MEMTRACK_METADATA_CURRENT_VERSION, + integration, + artifacts: saved.artifacts, + } + .save_to(profile_folder) +} + +/// Read every mapping artifact in the folder. One is written per tracked root +/// process, so a run with several of them contributes several files. +fn read_mappings(results_folder: &Path) -> Result> { + let suffix = format!(".{}.msgpack", MemtrackMappings::name()); + + let mut mappings = Vec::new(); + for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { + if !entry.file_name().to_string_lossy().ends_with(&suffix) { + continue; + } + + let file = std::fs::File::open(entry.path())?; + let artifact = MemtrackMappings::decode_from_reader(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?; + mappings.extend(artifact.mappings); + } + Ok(mappings) +} + +fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { + let mut loaded_modules = HashMap::::new(); + + for mapping in mappings { + let path = PathBuf::from(&mapping.path); + if !names_mapped_file(mapping, &path) { + continue; + } + + let load_bias = match ModuleSymbols::compute_load_bias( + &path, + mapping.avma_range.start, + mapping.avma_range.end, + mapping.file_offset, + ) { + Ok(load_bias) => load_bias, + Err(e) => { + debug!("Failed to compute load bias for {}: {e}", mapping.path); + continue; + } + }; + + let loaded_module = loaded_modules.entry(path.clone()).or_default(); + + if loaded_module.module_symbols.is_none() { + match ModuleSymbols::from_elf(&path) { + Ok(symbols) => loaded_module.module_symbols = Some(symbols), + Err(e) => debug!("Failed to load symbols for {}: {e}", mapping.path), + } + } + + // The ELF-derived halves are per file, the mounting is per mapping, so + // only the latter is recomputed for a module mapped more than once. + let unwind_data = match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + process_unwind_data.timestamp = Some(mapping.timestamp); + Some((unwind_data, process_unwind_data)) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } + }; + + let process_loaded_module = loaded_module + .process_loaded_modules + .entry(mapping.pid) + .or_default(); + process_loaded_module.symbols_load_bias = Some(load_bias); + + if let Some((unwind_data, process_unwind_data)) = unwind_data { + loaded_module.unwind_data = Some(unwind_data); + process_loaded_module.process_unwind_data = Some(process_unwind_data); + } + } + + loaded_modules +} + +/// Whether the path still names the file that was mapped. +/// +/// The mapping records the inode the kernel resolved the path from; a file +/// rebuilt or replaced since then is a different inode, and reading unwind data +/// out of it would bind eh_frame from the wrong binary to those addresses. +fn names_mapped_file(mapping: &ProcessMapping, path: &Path) -> bool { + let Ok(metadata) = std::fs::metadata(path) else { + debug!("{} is no longer readable", mapping.path); + return false; + }; + + // The recorded `dev` is the kernel's s_dev encoding, `st_dev` glibc's, so + // only the decomposed major/minor pair is comparable. + let recorded = (mapping.dev >> 20, mapping.dev & 0xF_FFFF, mapping.ino); + let current = ( + u64::from(libc::major(metadata.dev())), + u64::from(libc::minor(metadata.dev())), + metadata.ino(), + ); + + if recorded != current { + debug!( + "{} changed since it was mapped (recorded {recorded:?}, now {current:?})", + mapping.path + ); + return false; + } + true +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { + ProcessMapping { + pid: 42, + path: path.to_string(), + dev, + ino, + file_offset: 0, + avma_range: 0x1000..0x2000, + timestamp: 7, + } + } + + fn s_dev_of(path: &str) -> (u64, u64) { + let metadata = std::fs::metadata(path).unwrap(); + let dev = + u64::from(libc::major(metadata.dev())) << 20 | u64::from(libc::minor(metadata.dev())); + (dev, metadata.ino()) + } + + /// The recorded s_dev encoding and `st_dev` differ, so the check has to + /// decompose both or it rejects every module that did not change. + #[test] + fn accepts_a_file_that_still_has_the_recorded_inode() { + let path = "/proc/self/exe"; + let (dev, ino) = s_dev_of(path); + + assert!(names_mapped_file( + &mapping_for(path, dev, ino), + Path::new(path) + )); + } + + #[test] + fn rejects_a_file_whose_inode_changed() { + let path = "/proc/self/exe"; + let (dev, _) = s_dev_of(path); + + assert!(!names_mapped_file( + &mapping_for(path, dev, 0), + Path::new(path) + )); + } + + #[test] + fn rejects_a_path_that_no_longer_exists() { + let path = "/nonexistent/module.so"; + + assert!(!names_mapped_file( + &mapping_for(path, 1, 2), + Path::new(path) + )); + } + + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { + const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; + + let profile = tempfile::tempdir().unwrap(); + let results = profile.path().join("results"); + std::fs::create_dir_all(&results).unwrap(); + + let (dev, ino) = s_dev_of(MODULE); + MemtrackMappings { + mappings: vec![ProcessMapping { + pid: 1234, + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + avma_range: 0x5555_555a_7000..0x5555_556b_0000, + timestamp: 999, + }], + } + .save_with_pid_to(&results, 1234) + .unwrap(); + + save_module_artifacts( + profile.path(), + &results, + ("codspeed-rust".to_string(), "4.2.0".to_string()), + ) + .unwrap(); + + let metadata = MemtrackMetadata::from_reader( + std::fs::File::open(profile.path().join("memtrack.metadata")).unwrap(), + ) + .unwrap(); + + assert_eq!(metadata.version, MEMTRACK_METADATA_CURRENT_VERSION); + assert_eq!( + metadata.artifacts.mapped_process_module_symbols[&1234].len(), + 1 + ); + + let unwind = &metadata.artifacts.mapped_process_unwind_data_by_pid[&1234][0]; + assert_eq!(unwind.inner.timestamp, Some(999)); + assert!( + profile + .path() + .join(format!("{}.unwind_data", unwind.unwind_data_key)) + .exists() + ); + assert_eq!( + metadata.artifacts.path_key_to_path[&unwind.unwind_data_key], + PathBuf::from(MODULE) + ); + } +} From 3c5b5a2bfd09842efa892ec1d1ddfc0d14bdf9c8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:38:57 +0200 Subject: [PATCH 09/18] docs(memtrack): describe the mapping recorder --- crates/memtrack/AGENTS.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 0c8d86d41..2439bf0f0 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -20,11 +20,20 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed. +Mapping collection (`src/perf_mappings.rs`) uses Linux's native per-CPU perf event stream, not an LSM/BPF availability gate. `PerfMappingPoller` opens a `PERF_TYPE_SOFTWARE` dummy event with `PERF_ATTR_INHERIT` and `PERF_ATTR_MMAP2` on every online CPU for the tracked process, mmaps a perf ring per CPU, and drains those rings on a poll thread. It keeps executable mappings with absolute paths from `PERF_RECORD_MMAP2` and emits the single artifact representation, `MemtrackEventKind::Mapping` inside `MemtrackArtifact.events`, carrying the mapping's pid/tid/timestamp/address/path/device/inode/file offset/length. Opening or enabling any perf event requires the host's perf permissions (for example an allowed `perf_event_paranoid` policy or `CAP_PERFMON`); a permission error is returned from `Tracker::spawn` rather than silently disabling mapping collection. Kernel `PERF_RECORD_LOST` records, ring overruns, and malformed records increment the shared mapping-loss counter. `Tracker::dropped_events_count()` includes that counter with BPF ring-buffer drops, and `codspeed-memtrack track` aborts when the total is non-zero because the artifact is incomplete.` + +### Event stream compatibility + +Session relies on Rust's declaration-order field drop: _poller, _stack_poller, then _perf_mapping_poller. The BPF event and stack pollers therefore disconnect, fully drain, and join before the perf poller is dropped. PerfMappingPoller buffers mapping records and emits them during shutdown, after ordinary allocation/RSS/stack events have reached encode_events; encode_events preserves input order, so Mapping records are a terminal suffix in the one artifact stream. + +This ordering is compatibility-critical. Mapping is a newer event variant; older stream consumers may treat the first unknown Mapping as EOF. Keeping it as the suffix lets those consumers process the complete memory timeline before stopping at that first unknown record. Do not reorder the poller fields or emit mapping records before shutdown. + > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking. ## Key Directories -- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `stacks/`, `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/stack_capture.bpf.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/perf_mappings.rs` — native per-CPU `PERF_RECORD_MMAP2` collector. - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`. - `tests/` — integration tests + `snapshots/` (insta). - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces). From 1ef91af2fba2285ba26a3776470b4a536a93c903 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 12:40:52 +0200 Subject: [PATCH 10/18] test(memtrack): add nested allocation fixtures --- crates/memtrack/testdata/nested_doubling.c | 44 +++++++++++++++++++ .../testdata/nested_doubling_shared_free.c | 44 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 crates/memtrack/testdata/nested_doubling.c create mode 100644 crates/memtrack/testdata/nested_doubling_shared_free.c diff --git a/crates/memtrack/testdata/nested_doubling.c b/crates/memtrack/testdata/nested_doubling.c new file mode 100644 index 000000000..57badf398 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Each level allocates twice as much as its caller, then frees on the way back + * up, so the free order is the reverse of the allocation order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(1024) <-- free(2048) <-- free(4096) + * + * Every malloc and every free sits at a distinct call depth, so the six events + * also carry six distinct allocation stacks. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void) { + void* p = malloc(4096); + sleep(1); + escaped_pointer = p; + free(p); +} + +__attribute__((noinline)) static void level2(void) { + void* p = malloc(2048); + escaped_pointer = p; + sleep(1); + level3(); + free(p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + sleep(1); + level2(); + free(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/testdata/nested_doubling_shared_free.c b/crates/memtrack/testdata/nested_doubling_shared_free.c new file mode 100644 index 000000000..66c53fd99 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling_shared_free.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Same doubling allocation chain as nested_doubling.c, but ownership is handed + * down and the innermost level frees all three buffers in reverse order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(4096) + * free(2048) + * free(1024) + * + * The three mallocs come from three different call depths while all three frees + * share one, so a deallocation event must be attributed to the free site rather + * than to wherever its allocation happened. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void* outer, void* middle) { + void* p = malloc(4096); + escaped_pointer = p; + free(p); + free(middle); + free(outer); +} + +__attribute__((noinline)) static void level2(void* outer) { + void* p = malloc(2048); + escaped_pointer = p; + level3(outer, p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + level2(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} From 3f30d7101703c1126ad97f2e8de413494cd577e7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 12:41:18 +0200 Subject: [PATCH 11/18] test(memtrack): assert nested stack identities --- crates/memtrack/tests/stack_tests.rs | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index ae3e30f28..28950dd72 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -197,3 +197,87 @@ fn explicit_disable_suppresses_stack_capture() -> Result<(), Box Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + for (name, source) in [ + ( + "nested_doubling", + include_str!("../testdata/nested_doubling.c"), + ), + ( + "nested_doubling_shared_free", + include_str!("../testdata/nested_doubling_shared_free.c"), + ), + ] { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; + + let allocations: Vec<(u64, u64, u64)> = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { size, stack_hash } if (1024..=4096).contains(&size) => { + Some((size, e.addr, stack_hash)) + } + _ => None, + }) + .collect(); + + assert_eq!( + allocations + .iter() + .map(|(size, ..)| *size) + .collect::>(), + vec![1024, 2048, 4096], + "[{name}] each level must allocate twice its caller, outermost first" + ); + + // Both probes of one free report the same address, so dedup by address + // to recover the order the fixture released its buffers in. + let mut released: Vec = Vec::new(); + let mut free_hashes: Vec = Vec::new(); + for event in &events { + let MemtrackEventKind::Free { stack_hash } = event.kind else { + continue; + }; + if released.last() == Some(&event.addr) { + continue; + } + released.push(event.addr); + free_hashes.push(stack_hash); + } + + let allocated: Vec = allocations.iter().map(|(_, addr, _)| *addr).collect(); + let expected: Vec = allocated.iter().rev().copied().collect(); + assert_eq!( + released, expected, + "[{name}] buffers must be freed in the reverse of their allocation order" + ); + + let alloc_hashes: HashSet = allocations.iter().map(|(.., hash)| *hash).collect(); + assert_eq!( + alloc_hashes.len(), + allocations.len(), + "[{name}] each allocation depth must carry its own stack identity" + ); + assert!( + !alloc_hashes.contains(&0) && !free_hashes.contains(&0), + "[{name}] every allocation and free must carry a captured stack" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + } + Ok(()) +} From 42ff073182d4fcc30877af363e28a8cf2590d22d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 20:42:02 +0200 Subject: [PATCH 12/18] refactor(memtrack): capture module mappings with perf Replace the BPF LSM path cache and mapping ring with inherited per-CPU PERF_RECORD_MMAP2 collectors. Store executable mappings as a terminal suffix in the main memtrack stream so existing timeline consumers remain compatible, then extract and order them in the runner before generating module artifacts. --- Cargo.lock | 10 + Cargo.toml | 1 + crates/memtrack/AGENTS.md | 2 +- crates/memtrack/Cargo.toml | 1 + crates/memtrack/src/ebpf/c/attach.h | 5 + crates/memtrack/src/ebpf/c/event.h | 21 - crates/memtrack/src/ebpf/c/main.bpf.c | 1 - crates/memtrack/src/ebpf/c/mappings.bpf.h | 161 ------ crates/memtrack/src/ebpf/mappings/mod.rs | 7 - crates/memtrack/src/ebpf/mappings/records.rs | 83 --- crates/memtrack/src/ebpf/mappings/resolve.rs | 73 --- crates/memtrack/src/ebpf/mappings/support.rs | 103 ---- crates/memtrack/src/ebpf/memtrack/maps.rs | 52 -- crates/memtrack/src/ebpf/memtrack/mod.rs | 87 +--- crates/memtrack/src/ebpf/memtrack/tracking.rs | 23 - crates/memtrack/src/ebpf/mod.rs | 4 +- crates/memtrack/src/ebpf/tracker.rs | 183 +++---- crates/memtrack/src/lib.rs | 2 + crates/memtrack/src/perf_mappings.rs | 437 ++++++++++++++++ crates/memtrack/src/session.rs | 13 +- crates/memtrack/tests/c_tests.rs | 6 +- crates/memtrack/tests/shared.rs | 45 +- ...ng_shared_free_stack_capture_disabled.snap | 12 + ...ested_doubling_stack_capture_disabled.snap | 12 + ...__stack_paths_stack_capture_disabled.snap} | 0 crates/memtrack/tests/stack_budget_tests.rs | 22 + crates/memtrack/tests/stack_tests.rs | 360 ++++--------- .../src/artifacts/memtrack/mappings.rs | 37 -- .../src/artifacts/memtrack/mod.rs | 32 +- src/executor/memory/module_artifacts.rs | 477 ++++++++++++++++-- 30 files changed, 1204 insertions(+), 1068 deletions(-) delete mode 100644 crates/memtrack/src/ebpf/c/mappings.bpf.h delete mode 100644 crates/memtrack/src/ebpf/mappings/mod.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/records.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/resolve.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/support.rs create mode 100644 crates/memtrack/src/perf_mappings.rs create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap rename crates/memtrack/tests/snapshots/{stack_tests__stack_capture_disabled.snap => stack_tests__stack_paths_stack_capture_disabled.snap} (100%) create mode 100644 crates/memtrack/tests/stack_budget_tests.rs delete mode 100644 crates/runner-shared/src/artifacts/memtrack/mappings.rs diff --git a/Cargo.lock b/Cargo.lock index 8e555476f..4956c2d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2402,6 +2402,7 @@ dependencies = [ "object", "parking_lot", "paste", + "perf-event-open-sys", "rayon", "rstest", "runner-shared", @@ -2813,6 +2814,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perf-event-open-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5f8d1487a4ffa23c80a1c355dd27235f9b66fb71ba0f261eb417e4fe8451347" +dependencies = [ + "libc", +] + [[package]] name = "pest" version = "2.8.6" diff --git a/Cargo.toml b/Cargo.toml index 8bd28f039..34be934d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ ipc-channel = "0.20" itertools = "0.14.0" rayon = "1.12" linux-perf-event-reader = "0.10.2" # matches the version linux-perf-data resolves to +perf-event-open-sys = "6.0" env_logger = "0.11.10" tempfile = "3.27.0" object = { version = "0.39", default-features = false, features = ["read_core", "elf"] } diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 2439bf0f0..f159b879e 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -84,7 +84,7 @@ sudo -E cargo test --test c_tests -- --test-threads 1 - **Build toolchain:** `clang` + BTF/vmlinux headers, `libbpf-dev`, `zlib1g-dev`, `pkgconf`, `build-essential`; vendored libbpf also needs `autopoint`/`bison`/`flex`. - `vmlinux.h` is pinned to a specific git rev; `libbpf-rs` uses the `vendored` feature (dist links `libbpf-rs/static`). -Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate). +Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_MEMTRACK_TRACK_ALLOCATORS` (0/false disables), `CODSPEED_MEMTRACK_TRACK_RMAP` (1 enables), `CODSPEED_MEMTRACK_CAPTURE_STACKS` (0/false disables), `CODSPEED_MEMTRACK_STACK_BUDGET` (stack copy size in bytes, default 8192), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate). ## Testing & QA diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 4912802da..68da1218f 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -34,6 +34,7 @@ itertools = { workspace = true } paste = "1.0.15" libbpf-rs = { version = "0.26", features = ["vendored"], optional = true } object = { workspace = true } +perf-event-open-sys = { workspace = true } rayon = "1.12" parking_lot = "0.12" typed-builder = "0.23.2" diff --git a/crates/memtrack/src/ebpf/c/attach.h b/crates/memtrack/src/ebpf/c/attach.h index 90cbe4360..e188c7d5f 100644 --- a/crates/memtrack/src/ebpf/c/attach.h +++ b/crates/memtrack/src/ebpf/c/attach.h @@ -14,6 +14,11 @@ #define MEMTRACK_PROT_EXEC 0x4 #define MEMTRACK_SIGSTOP 19 +struct inode_key { + __u64 dev; + __u64 ino; +}; + /* (dev, ino) -> 1; populated by userspace after classify/attach */ BPF_HASH_MAP(known_inodes, struct inode_key, __u8, 8192); /* Requests are 24 B and rare; overflow aborts the run via the counter below */ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index e3fdab0aa..7e10a7756 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -115,13 +115,6 @@ struct event { } data; }; -/* Identifies a mapped file across both the attach watcher and the mapping - * recorder. `dev` uses the kernel's s_dev encoding: (major << 20) | minor. */ -struct inode_key { - uint64_t dev; - uint64_t ino; -}; - /* Request from the exec-mapping watcher to the userspace attach worker */ struct attach_request { uint32_t pid; @@ -129,18 +122,4 @@ struct attach_request { uint64_t ino; }; -/* One executable file mapping, mirroring PERF_RECORD_MMAP2. The path is not - * here: it is resolved once per inode into a BPF map that userspace joins - * against, since every mapping of the same file shares it. */ -struct mapping_record { - uint64_t dev; - uint64_t ino; - uint64_t file_offset; /* offset of the mapping's first byte in the file */ - uint64_t start; - uint64_t end; - uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ - uint32_t pid; - uint32_t _pad; -}; - #endif /* __EVENT_H__ */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 7a068c605..b405f572b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,7 +8,6 @@ #include "allocator.h" #include "attach.h" #include "event.h" -#include "mappings.bpf.h" #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h deleted file mode 100644 index 5b5635a12..000000000 --- a/crates/memtrack/src/ebpf/c/mappings.bpf.h +++ /dev/null @@ -1,161 +0,0 @@ -#ifndef __MAPPINGS_BPF_H__ -#define __MAPPINGS_BPF_H__ - -#include "event.h" -#include "utils/folio.h" -#include "utils/map_helpers.h" -#include "utils/process_tracking.h" - -/* == Mapping recorder == - * - * Reconstructs what `PERF_RECORD_MMAP2` gives perf: which file a tracked - * process mapped, where, so raw stack addresses can be attributed to modules - * offline. No single hook carries both halves: - * - * security_mmap_file(file, ..) has the file, runs before the VMA exists - * perf_event_mmap(vma) has the addresses, cannot resolve a path - * - * The path therefore lands in a per-inode cache, and the address-bearing hook - * emits inode-keyed records that userspace joins against that cache while this - * BPF object is still loaded. - * - * Path resolution is only reachable from an LSM program: `bpf_d_path()` is - * restricted to sleepable LSM hooks, `BPF_TRACE_ITER` and an fentry allowlist - * holding no mmap path, and the newer `bpf_path_d_path()` kfunc rejects - * non-LSM program types. Both variants are compiled; userspace autoloads the - * one the running kernel supports and neither when the bpf LSM is inactive. */ - -/* VM_EXEC from linux/mm.h, which vmlinux.h does not carry (it is a macro, not a - * type). Only executable mappings are recorded: unwind data and symbols are - * looked up by text address. */ -#define MEMTRACK_VM_EXEC 0x00000004 - -/* d_path() fails with -ENAMETOOLONG rather than truncating, so a short buffer - * loses whole modules. PATH_MAX keeps that from happening. */ -#define MEMTRACK_MAX_PATH 4096 - -struct inode_path { - __u32 len; /* bytes written by d_path, including the NUL */ - char path[MEMTRACK_MAX_PATH]; -}; - -/* Every mapping of an inode shares its cached path. */ -BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); - -/* A dropped record may leave a module unresolved. */ -BPF_RINGBUF(mappings, 256 * 1024); -BPF_ARRAY_MAP(mapping_dropped, __u64, 1); - -/* The path does not fit on the BPF stack; build it in this per-CPU scratch map. */ -struct { - __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, struct inode_path); -} path_scratch SEC(".maps"); - -extern int bpf_path_d_path(const struct path* path, char* buf, __u64 buf__sz) __ksym __weak; - -static __always_inline void bump_mapping_dropped(void) { - __u32 zero = 0; - __u64* drops = bpf_map_lookup_elem(&mapping_dropped, &zero); - if (drops) { - __sync_fetch_and_add(drops, 1); - } -} - -/* Return a scratch slot when this inode has no cached path. */ -static __always_inline struct inode_path* mapping_path_slot(struct file* file, - struct inode_key* key) { - if (!file || !is_tracked(current_tgid())) { - return NULL; - } - - key->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); - key->ino = BPF_CORE_READ(file, f_inode, i_ino); - if (bpf_map_lookup_elem(&path_by_inode, key)) { - return NULL; - } - - __u32 zero = 0; - return bpf_map_lookup_elem(&path_scratch, &zero); -} - -/* Publish a resolved path. A failed resolution is not cached, so the next - * mapping of the same inode retries instead of losing the module for the run. */ -static __always_inline void commit_mapping_path(struct inode_key* key, struct inode_path* entry, - int len) { - if (len <= 0) { - return; - } - entry->len = (__u32)len; - bpf_map_update_elem(&path_by_inode, key, entry, BPF_NOEXIST); -} - -/* Kernels >= 6.12: the kfunc is callable from any LSM program. */ -SEC("lsm/mmap_file") -int BPF_PROG(cache_mmap_path_kfunc, struct file* file, unsigned long reqprot, unsigned long prot, - unsigned long flags) { - struct inode_key key = {}; - struct inode_path* entry = mapping_path_slot(file, &key); - if (entry) { - commit_mapping_path(&key, entry, - bpf_path_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); - } - return 0; -} - -/* Kernels 5.11..6.11: `bpf_d_path()` needs a sleepable LSM hook, which - * `mmap_file` has been since 5.11. */ -SEC("lsm.s/mmap_file") -int BPF_PROG(cache_mmap_path_legacy, struct file* file, unsigned long reqprot, unsigned long prot, - unsigned long flags) { - struct inode_key key = {}; - struct inode_path* entry = mapping_path_slot(file, &key); - if (entry) { - commit_mapping_path(&key, entry, bpf_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); - } - return 0; -} - -/* The same hook perf emits MMAP2 from, so the recorded geometry matches what - * the walltime pipeline already consumes: the file offset is in bytes, not - * pages. */ -SEC("fentry/perf_event_mmap") -int BPF_PROG(record_mmap, struct vm_area_struct* vma) { - if (!vma) { - return 0; - } - - __u32 tgid = current_tgid(); - if (!is_tracked(tgid)) { - return 0; - } - - struct file* file = BPF_CORE_READ(vma, vm_file); - if (!file) { - return 0; - } - if (!(BPF_CORE_READ(vma, vm_flags) & MEMTRACK_VM_EXEC)) { - return 0; - } - - struct mapping_record* rec = bpf_ringbuf_reserve(&mappings, sizeof(*rec), 0); - if (!rec) { - bump_mapping_dropped(); - return 0; - } - - rec->pid = tgid; - rec->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); - rec->ino = BPF_CORE_READ(file, f_inode, i_ino); - rec->file_offset = (__u64)BPF_CORE_READ(vma, vm_pgoff) << page_shift; - rec->start = BPF_CORE_READ(vma, vm_start); - rec->end = BPF_CORE_READ(vma, vm_end); - rec->timestamp = bpf_ktime_get_ns(); - bpf_ringbuf_submit(rec, 0); - - return 0; -} - -#endif /* __MAPPINGS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/mappings/mod.rs b/crates/memtrack/src/ebpf/mappings/mod.rs deleted file mode 100644 index 581de916b..000000000 --- a/crates/memtrack/src/ebpf/mappings/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod records; -mod resolve; -mod support; - -pub(crate) use records::MappingRecord; -pub(crate) use resolve::resolve_mappings; -pub use support::MappingSupport; diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs deleted file mode 100644 index 4c660031e..000000000 --- a/crates/memtrack/src/ebpf/mappings/records.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::ebpf::events::bindings::mapping_record; - -/// One executable file mapping as the BPF recorder saw it. The path is resolved -/// separately, per inode. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct MappingRecord { - pub pid: u32, - pub dev: u64, - pub ino: u64, - pub file_offset: u64, - pub start: u64, - pub end: u64, - pub timestamp: u64, -} - -impl MappingRecord { - pub fn parse(data: &[u8]) -> Option { - if data.len() < std::mem::size_of::() { - return None; - } - - // SAFETY: the length is checked above, and the layout is the - // bindgen-generated C ABI struct. - let record: mapping_record = unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; - Some(Self { - pid: record.pid, - dev: record.dev, - ino: record.ino, - file_offset: record.file_offset, - start: record.start, - end: record.end, - timestamp: record.timestamp, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn encode(record: mapping_record) -> Vec { - // SAFETY: reading a plain-data struct as bytes. - unsafe { - std::slice::from_raw_parts( - (&record as *const mapping_record).cast::(), - std::mem::size_of::(), - ) - } - .to_vec() - } - - #[test] - fn well_formed_record_round_trips_every_field() { - let bytes = encode(mapping_record { - dev: 0x1_0002, - ino: 4242, - file_offset: 0x2000, - start: 0x5555_5555_0000, - end: 0x5555_5556_0000, - timestamp: 987_654_321, - pid: 7, - _pad: 0, - }); - - assert_eq!( - MappingRecord::parse(&bytes), - Some(MappingRecord { - pid: 7, - dev: 0x1_0002, - ino: 4242, - file_offset: 0x2000, - start: 0x5555_5555_0000, - end: 0x5555_5556_0000, - timestamp: 987_654_321, - }) - ); - } - - #[test] - fn truncated_buffer_returns_none() { - assert!(MappingRecord::parse(&[0u8; 8]).is_none()); - } -} diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs deleted file mode 100644 index 9c20ba7f1..000000000 --- a/crates/memtrack/src/ebpf/mappings/resolve.rs +++ /dev/null @@ -1,73 +0,0 @@ -use super::MappingRecord; -use crate::prelude::*; -use runner_shared::artifacts::ProcessMapping; -use std::collections::HashMap; - -/// Records without a path are dropped because their unwind data and symbols cannot be read. -pub(crate) fn resolve_mappings( - records: Vec, - paths: &HashMap<(u64, u64), String>, -) -> Vec { - let mut unresolved = 0; - let mappings = records - .into_iter() - .filter_map(|record| { - let Some(path) = paths.get(&(record.dev, record.ino)) else { - unresolved += 1; - return None; - }; - - Some(ProcessMapping { - pid: record.pid as i32, - path: path.clone(), - dev: record.dev, - ino: record.ino, - file_offset: record.file_offset, - avma_range: record.start..record.end, - timestamp: record.timestamp, - }) - }) - .collect(); - - if unresolved > 0 { - debug!("{unresolved} mapping records had no resolved path and were dropped"); - } - mappings -} - -#[cfg(test)] -mod tests { - use super::*; - - fn record(dev: u64, ino: u64) -> MappingRecord { - MappingRecord { - pid: 5, - dev, - ino, - file_offset: 0x1000, - start: 0x4000, - end: 0x8000, - timestamp: 42, - } - } - - #[test] - fn resolves_records_against_the_path_cache() { - let paths = HashMap::from([((1, 2), "/lib/libc.so.6".to_string())]); - - let mappings = resolve_mappings(vec![record(1, 2)], &paths); - - assert_eq!(mappings.len(), 1); - assert_eq!(mappings[0].path, "/lib/libc.so.6"); - assert_eq!(mappings[0].avma_range, 0x4000..0x8000); - assert_eq!(mappings[0].file_offset, 0x1000); - assert_eq!(mappings[0].pid, 5); - } - - /// A module we cannot name is a module we cannot read, so it must not reach - /// the artifact as an empty path. - #[test] - fn drops_records_without_a_resolved_path() { - assert!(resolve_mappings(vec![record(9, 9)], &HashMap::new()).is_empty()); - } -} diff --git a/crates/memtrack/src/ebpf/mappings/support.rs b/crates/memtrack/src/ebpf/mappings/support.rs deleted file mode 100644 index 9e2f3a190..000000000 --- a/crates/memtrack/src/ebpf/mappings/support.rs +++ /dev/null @@ -1,103 +0,0 @@ -use crate::kernel::KernelVersion; -use crate::prelude::*; - -/// How the running kernel can resolve a mapped file's path inside BPF. -/// -/// Only a BPF LSM program can do it at all: `bpf_d_path()` is restricted to -/// `BPF_TRACE_ITER` programs, sleepable LSM hooks and a fixed fentry allowlist -/// that contains no mmap path (`bpf_d_path_allowed()` in -/// `kernel/trace/bpf_trace.c`), and the `bpf_path_d_path()` kfunc that replaces -/// it rejects every program type but LSM (`bpf_fs_kfuncs_filter()` in -/// `fs/bpf_fs_kfuncs.c`). -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum MappingSupport { - /// Paths cannot be resolved, so allocation stacks could not be attributed to - /// modules and are not worth capturing. - Unsupported, - /// Sleepable LSM hook calling `bpf_d_path()` (kernel >= 5.11). - Legacy, - /// LSM hook calling the `bpf_path_d_path()` kfunc (kernel >= 6.12). - Kfunc, -} - -impl MappingSupport { - /// What the running kernel and its boot configuration provide. - /// - /// The kernel release is only half the gate: `bpf` must also be in the - /// active LSM list, which is fixed at boot by `CONFIG_LSM`/`lsm=` and cannot - /// be inferred from the version. - pub fn detect() -> Self { - if !bpf_lsm_active() { - info!( - "The bpf LSM is not active (see /sys/kernel/security/lsm), so mapped module paths \ - cannot be resolved" - ); - return Self::Unsupported; - } - - let version = match KernelVersion::current() { - Ok(version) => version, - Err(e) => { - warn!("Failed to read the kernel version, no mapping records: {e:#}"); - return Self::Unsupported; - } - }; - - let support = Self::for_version(version); - match support { - Self::Unsupported => { - info!("Kernel {version} cannot resolve paths from an LSM program (needs >= 5.11)") - } - Self::Legacy => { - debug!("Kernel {version} predates the bpf_path_d_path kfunc, using bpf_d_path") - } - Self::Kfunc => {} - } - support - } - - fn for_version(version: KernelVersion) -> Self { - if version < KernelVersion::new(5, 11) { - return Self::Unsupported; - } - if version < KernelVersion::new(6, 12) { - return Self::Legacy; - } - Self::Kfunc - } -} - -/// Whether `bpf` is one of the LSMs the running kernel initialized. An -/// unreadable file means securityfs is not mounted, in which case no LSM program -/// will attach either. -fn bpf_lsm_active() -> bool { - const PATH: &str = "/sys/kernel/security/lsm"; - - let Ok(active) = std::fs::read_to_string(PATH) else { - debug!("Could not read {PATH} to check whether the bpf LSM is active"); - return false; - }; - active.trim().split(',').any(|lsm| lsm == "bpf") -} - -#[cfg(test)] -mod tests { - use super::*; - - /// `bpf_lsm_mmap_file` has been in the sleepable hook set since 5.11, and - /// 6.12 is the first release carrying `bpf_path_d_path`. - #[test] - fn maps_releases_to_support_levels() { - for (major, minor, expected) in [ - (5, 4, MappingSupport::Unsupported), - (5, 10, MappingSupport::Unsupported), - (5, 11, MappingSupport::Legacy), - (6, 11, MappingSupport::Legacy), - (6, 12, MappingSupport::Kfunc), - (7, 1, MappingSupport::Kfunc), - ] { - let version = KernelVersion::new(major, minor); - assert_eq!(MappingSupport::for_version(version), expected, "{version}"); - } - } -} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index fa8f6f920..994650d53 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -2,7 +2,6 @@ use super::MemtrackBpf; use crate::ebpf::stacks::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; -use std::collections::HashMap; impl MemtrackBpf { pub fn add_tracked_pid(&mut self, pid: i32) -> Result<()> { @@ -63,39 +62,6 @@ impl MemtrackBpf { ) } - /// Number of mapping records dropped because their ring buffer was full. - /// A non-zero value means a module may be missing from the trace. - pub fn mapping_dropped_count(&self) -> Result { - read_counter( - with_skel!(self, skel => &skel.maps.mapping_dropped), - "mapping_dropped", - ) - } - - /// The paths the kernel resolved for every mapped file, keyed by - /// `(dev, ino)`. Only readable while the BPF object is alive. - pub fn mapped_paths(&self) -> Result> { - let map = with_skel!(self, skel => &skel.maps.path_by_inode); - - let mut paths = HashMap::new(); - for key in map.keys() { - let Some(value) = map - .lookup(&key, libbpf_rs::MapFlags::ANY) - .context("Failed to read a resolved mapping path")? - else { - continue; - }; - - let Some((dev, ino)) = inode_key(&key) else { - continue; - }; - if let Some(path) = inode_path(&value) { - paths.insert((dev, ino), path); - } - } - Ok(paths) - } - pub fn dropped_events_count(&self) -> Result { read_counter( with_skel!(self, skel => &skel.maps.dropped_events), @@ -152,24 +118,6 @@ fn le(bytes: &[u8]) -> u64 { .fold(0, |acc, &b| acc << 8 | u64::from(b)) } -/// Split a `struct inode_key { __u64 dev; __u64 ino; }` map key. -fn inode_key(key: &[u8]) -> Option<(u64, u64)> { - if key.len() < 16 { - return None; - } - Some((le(&key[..8]), le(&key[8..16]))) -} - -/// Read a `struct inode_path { __u32 len; char path[]; }` map value. The kernel -/// wrote `len` bytes including the NUL terminator. -fn inode_path(value: &[u8]) -> Option { - const PATH_OFFSET: usize = 4; - - let len = u32::from_le_bytes(value.get(..PATH_OFFSET)?.try_into().ok()?) as usize; - let path = value.get(PATH_OFFSET..PATH_OFFSET + len.saturating_sub(1))?; - Some(String::from_utf8_lossy(path).into_owned()) -} - /// Read slot 0 of a single-entry `__u64` array map. fn read_counter(map: &impl MapCore, name: &str) -> Result { let key = 0u32; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index c3670c6ea..19cd898f3 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,8 +6,8 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; -use crate::ebpf::mappings::MappingSupport; use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::tracker::TrackerOptions; mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); @@ -120,38 +120,24 @@ pub struct MemtrackBpf { pub(super) skel: Skel, pub(super) probes: Vec, rmap: RmapSupport, - pub(super) mappings: MappingSupport, } impl MemtrackBpf { - /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap( - track_rmap: bool, - capture_stacks: bool, - mappings: MappingSupport, - ) -> Result { - let variant = if has_delegated_bpf_token() { - BpfVariant::Token - } else { - BpfVariant::Legacy - }; - Self::with_variant(variant, track_rmap, capture_stacks, mappings) - } - - /// Load a specific variant rather than the one [`Self::new_with_rmap`] - /// would detect. Either attaches given host privileges; the token only - /// matters when `bpf()` is called from an unprivileged user namespace. - /// - /// `capture_stacks` enables allocation stack capture, and `mappings` - /// selects the path-resolving LSM program the running kernel supports. - pub fn with_variant( - variant: BpfVariant, - track_rmap: bool, - capture_stacks: bool, - mappings: MappingSupport, - ) -> Result { + /// Load the skeleton using the requested configuration. + pub fn new(options: &TrackerOptions) -> Result { crate::kernel::KernelBtf::ensure_available()?; + let variant = options.variant.unwrap_or_else(|| { + if has_delegated_bpf_token() { + BpfVariant::Token + } else { + BpfVariant::Legacy + } + }); + let track_rmap = options.rmap; + let capture_stacks = options.stack_capture; + let stack_copy_budget = ((options.stack_budget / 512) * 512) + .clamp(512, crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY); let page_shift = page_shift()?; let rmap = if track_rmap { RmapSupport::detect() @@ -182,6 +168,7 @@ impl MemtrackBpf { } if capture_stacks { rodata.capture_stacks_enabled = 1; + rodata.stack_copy_budget = stack_copy_budget; } } @@ -216,26 +203,6 @@ impl MemtrackBpf { RmapSupport::CoreAndPud => {} } - // The kfunc variant fails to load on kernels without - // `bpf_path_d_path`, and neither LSM program can attach when the - // bpf LSM is inactive; without a path there is nothing to - // resolve records against, so the recorder goes too. - match mappings { - MappingSupport::Unsupported => { - open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); - open_skel.progs.cache_mmap_path_legacy.set_autoload(false); - open_skel.progs.record_mmap.set_autoload(false); - open_skel.maps.mappings.set_max_entries(4096)?; - open_skel.maps.path_by_inode.set_max_entries(1)?; - } - MappingSupport::Legacy => { - open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); - } - MappingSupport::Kfunc => { - open_skel.progs.cache_mmap_path_legacy.set_autoload(false); - } - } - $skel(Box::new( open_skel .load() @@ -257,7 +224,6 @@ impl MemtrackBpf { skel, probes: Vec::new(), rmap, - mappings, }) } @@ -282,7 +248,7 @@ impl MemtrackBpf { poll_interval_ms: u64, tx: std::sync::mpsc::Sender, ) -> Result { - use crate::ebpf::stacks::events; + use crate::ebpf::events; use runner_shared::artifacts::MemtrackEventKind; // The poller outlives this borrow of the skeleton, so the chain lookup @@ -323,27 +289,6 @@ impl MemtrackBpf { )) } - /// Poll the mapping-record ring buffer into `tx`. Same contract as - /// [`Self::poll_events_with_channel`]. - pub(crate) fn poll_mappings_with_channel( - &self, - poll_interval_ms: u64, - tx: std::sync::mpsc::Sender, - ) -> Result { - with_skel!(self, skel => RingBufferPoller::new( - &skel.maps.mappings, - crate::ebpf::mappings::MappingRecord::parse, - tx, - poll_interval_ms, - )) - } - - /// Whether the mapping recorder is loaded, i.e. whether its ring buffer is - /// worth polling. - pub fn records_mappings(&self) -> bool { - self.mappings != MappingSupport::Unsupported - } - /// Number of currently-attached probes/links. pub fn probe_count(&self) -> usize { self.probes.len() diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 350c039ee..3e00ceb3b 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -1,5 +1,4 @@ use super::{MemtrackBpf, RmapSupport}; -use crate::ebpf::mappings::MappingSupport; use crate::prelude::*; use paste::paste; @@ -65,26 +64,4 @@ impl MemtrackBpf { self.probes.push(link); Ok(()) } - - /// Attach the mapping recorder: the LSM hook caching resolved paths and the - /// `perf_event_mmap` fentry emitting the address records. Only the LSM - /// variant the running kernel supports was loaded. - pub fn attach_mapping_recorder(&mut self) -> Result<()> { - let link = match self.mappings { - MappingSupport::Unsupported => return Ok(()), - MappingSupport::Legacy => { - with_skel!(mut self, skel => skel.progs.cache_mmap_path_legacy.attach()) - } - MappingSupport::Kfunc => { - with_skel!(mut self, skel => skel.progs.cache_mmap_path_kfunc.attach()) - } - } - .context("Failed to attach the mmap path resolver")?; - self.probes.push(link); - - let link = with_skel!(mut self, skel => skel.progs.record_mmap.attach()) - .context("Failed to attach the mapping recorder")?; - self.probes.push(link); - Ok(()) - } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 743970c86..6cd589ab9 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -1,6 +1,5 @@ mod attach_worker; mod events; -pub(crate) mod mappings; mod memtrack; pub(crate) mod poller; mod proc_fs; @@ -8,9 +7,8 @@ mod spawn; mod stacks; mod tracker; -pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; -pub use stacks::counters::StackCaptureStats; +pub use stacks::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 13ed9145f..0ba4b9f22 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,22 +1,23 @@ use crate::ebpf::attach_worker::AttachWorker; -use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::stacks::config::stack_capture_from_env; -use crate::ebpf::stacks::counters::StackCaptureStats; +use crate::ebpf::stacks::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; -use runner_shared::artifacts::MemtrackMappings; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; #[derive(Debug, Clone, Copy, TypedBuilder)] pub struct TrackerOptions { + /// BPF attach mechanism, or automatic detection when unset. + #[builder(default)] + pub variant: Option, /// Attach allocator uprobes (malloc/free/calloc/...) through the /// exec-mapping watcher. #[builder(default = true)] @@ -27,6 +28,9 @@ pub struct TrackerOptions { /// Capture allocation call stacks. #[builder(default = true)] pub stack_capture: bool, + /// Maximum bytes of user stack to copy per captured call stack. + #[builder(default = 8192)] + pub stack_budget: u32, } impl TrackerOptions { @@ -37,21 +41,38 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .rmap(std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1")) - .stack_capture(stack_capture_from_env()) + .stack_capture(!matches!( + std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref(), + Ok("0") | Ok("false") + )) + .stack_budget( + std::env::var("CODSPEED_MEMTRACK_STACK_BUDGET") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8192), + ) .build() } } +impl Default for TrackerOptions { + fn default() -> Self { + Self::builder().build() + } +} + pub struct Tracker { bpf: Arc>, worker: Mutex>, - allocators: bool, - /// The dedup gate spans the whole BPF object, so a second session would - /// reference stack records the first one already consumed. - stacks_polled: Option, - /// Filled by the mapping poller; drained by [`Tracker::mappings`] after the - /// session is dropped, so the poller's final drain is included. - mapping_rx: Mutex>>, + options: TrackerOptions, + /// Number of native perf mapping records lost due to ring-buffer overflow. + mapping_lost: Arc, +} + +fn kill_and_wait(child: &mut std::process::Child) { + // Cleanup is best effort so the setup error remains the returned error. + let _ = child.kill(); + let _ = child.wait(); } impl Tracker { @@ -63,48 +84,23 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { - let mappings = MappingSupport::detect(); - - // Stacks are raw addresses: without mapped module paths nothing can - // attribute them, so capturing them would only inflate the artifact. - let capture_stacks = match (options.stack_capture, mappings) { - (true, MappingSupport::Unsupported) => { - warn!("Allocation stack capture needs in-kernel path resolution; disabling it"); - false - } - (capture_stacks, _) => capture_stacks, - }; - Self::build( - MemtrackBpf::new_with_rmap(options.rmap, capture_stacks, mappings)?, - options.allocators, - capture_stacks, - ) - } - /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of - /// the detected one. - pub fn with_variant(variant: BpfVariant) -> Result { - let track_rmap = TrackerOptions::from_env().rmap; - Self::build( - MemtrackBpf::with_variant(variant, track_rmap, false, MappingSupport::detect())?, - true, - false, - ) + let bpf = MemtrackBpf::new(&options)?; + Self::build(bpf, options) } /// Build a tracker: attach lifetime tracepoints (and rmap fentries when the /// skeleton was opened for them), plus, when `allocators` is set, the /// exec-mapping watcher and the on-demand allocator-attach worker. - fn build(mut bpf: MemtrackBpf, allocators: bool, capture_stacks: bool) -> Result { + fn build(mut bpf: MemtrackBpf, options: TrackerOptions) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; - if allocators { + if options.allocators { bpf.attach_exec_watcher()?; - bpf.attach_mapping_recorder()?; } let bpf = Arc::new(Mutex::new(bpf)); - let worker = if allocators { + let worker = if options.allocators { Some(AttachWorker::start(bpf.clone())?) } else { None @@ -113,9 +109,8 @@ impl Tracker { Ok(Self { bpf, worker: Mutex::new(worker), - allocators, - stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), - mapping_rx: Mutex::new(None), + options, + mapping_lost: Arc::new(AtomicU64::new(0)), }) } @@ -128,80 +123,60 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { - let capture_stacks = match &self.stacks_polled { - Some(polled) if polled.swap(true, Ordering::Relaxed) => { - bail!("stack capture supports a single spawned command per tracker") - } - Some(_) => true, - None => false, - }; + let capture_stacks = self.options.stack_capture; let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); } - let child = spawn_stopped(&mut wrapped)?; + let mut child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - match self.worker.lock().as_ref() { - Some(worker) => worker.set_root_pid(pid), - // No watcher to arm means exec mappings would be missed. - None if self.allocators => bail!("tracker already finished"), - None => {} - } + let setup = (|| -> Result<_> { + match self.worker.lock().as_ref() { + Some(worker) => worker.set_root_pid(pid), + // No watcher to arm means exec mappings would be missed. + None if self.options.allocators => bail!("tracker already finished"), + None => {} + } - let (tx, rx) = mpsc::channel(); - let (mapping_tx, mapping_rx) = mpsc::channel(); - let (poller, stack_poller, mapping_poller) = { - let mut bpf = self.bpf.lock(); - bpf.add_tracked_pid(pid)?; - let stack_poller = capture_stacks - .then(|| bpf.poll_stacks(10, tx.clone())) - .transpose()?; - let mapping_poller = bpf - .records_mappings() - .then(|| bpf.poll_mappings_with_channel(10, mapping_tx)) + let (tx, rx) = mpsc::channel(); + let (poller, stack_poller) = { + let mut bpf = self.bpf.lock(); + bpf.add_tracked_pid(pid)?; + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx.clone())?, stack_poller) + }; + let perf_mapping_poller = capture_stacks + .then(|| PerfMappingPoller::start(pid, tx, self.mapping_lost.clone())) .transpose()?; - ( - bpf.poll_events_with_channel(10, tx)?, - stack_poller, - mapping_poller, - ) + + Ok((rx, poller, stack_poller, perf_mapping_poller)) + })(); + let (rx, poller, stack_poller, perf_mapping_poller) = match setup { + Ok(pollers) => pollers, + Err(error) => { + kill_and_wait(&mut child); + return Err(error); + } }; - *self.mapping_rx.lock() = Some(mapping_rx); - resume(pid)?; + + if let Err(error) = resume(pid) { + kill_and_wait(&mut child); + return Err(error); + } Ok(Session::new( child, rx, poller, stack_poller, - mapping_poller, + perf_mapping_poller, )) } - - /// The module mappings recorded during the run, joined with the paths the - /// kernel resolved for them. Call after dropping the session so the poller's - /// final drain is included, and before the BPF object is torn down. - pub fn mappings(&self) -> Result { - let Some(rx) = self.mapping_rx.lock().take() else { - return Ok(MemtrackMappings::default()); - }; - - let records: Vec<_> = rx.try_iter().collect(); - let paths = self.bpf.lock().mapped_paths()?; - - let dropped = self.bpf.lock().mapping_dropped_count()?; - if dropped > 0 { - warn!("{dropped} mapping records were dropped; some modules may be unresolved"); - } - - Ok(MemtrackMappings { - mappings: resolve_mappings(records, &paths), - }) - } - /// Enable allocator-event tracking in the BPF program. Lifetime events /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids /// regardless of this toggle. @@ -217,7 +192,7 @@ impl Tracker { /// Number of events the kernel dropped because the ring buffer was full. /// A non-zero value means the resulting trace is incomplete. pub fn dropped_events_count(&self) -> Result { - self.bpf.lock().dropped_events_count() + Ok(self.bpf.lock().dropped_events_count()? + self.mapping_lost.load(Ordering::Relaxed)) } /// Per-cause counts of stack captures that were skipped or truncated. @@ -225,6 +200,10 @@ impl Tracker { self.bpf.lock().stack_capture_stats() } + pub fn stack_capture_enabled(&self) -> bool { + self.options.stack_capture + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() diff --git a/crates/memtrack/src/lib.rs b/crates/memtrack/src/lib.rs index ccd93399f..30cfcc99c 100644 --- a/crates/memtrack/src/lib.rs +++ b/crates/memtrack/src/lib.rs @@ -4,6 +4,8 @@ mod bpf_token; mod ebpf; mod ipc; mod kernel; +#[cfg(feature = "ebpf")] +mod perf_mappings; pub mod prelude; #[cfg(feature = "ebpf")] mod session; diff --git a/crates/memtrack/src/perf_mappings.rs b/crates/memtrack/src/perf_mappings.rs new file mode 100644 index 000000000..45350dbf4 --- /dev/null +++ b/crates/memtrack/src/perf_mappings.rs @@ -0,0 +1,437 @@ +use crate::prelude::*; +use perf_event_open_sys::bindings::{ + PERF_COUNT_SW_DUMMY, PERF_FLAG_FD_CLOEXEC, PERF_RECORD_LOST, PERF_RECORD_MMAP2, + PERF_SAMPLE_TID, PERF_SAMPLE_TIME, PERF_TYPE_SOFTWARE, perf_event_attr, perf_event_header, + perf_event_mmap_page, +}; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::io; +use std::mem::size_of; +use std::os::fd::RawFd; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::thread::JoinHandle; +use std::time::Duration; + +const DATA_PAGES: usize = 64; + +struct PerfRing { + fd: RawFd, + mapping: *mut u8, + mapping_len: usize, + data_offset: usize, + data_size: usize, + enabled: bool, +} + +// The mapping is exclusively consumed by the poll thread. +unsafe impl Send for PerfRing {} + +impl PerfRing { + fn open(pid: libc::pid_t, cpu: u32, page_size: usize) -> Result { + let mapping_len = page_size + .checked_mul(DATA_PAGES + 1) + .context("perf ring mapping size overflow")?; + ensure!( + mapping_len >= size_of::(), + "perf ring mapping is smaller than its metadata page" + ); + let mut attr = perf_event_attr { + type_: PERF_TYPE_SOFTWARE, + size: size_of::() as u32, + config: PERF_COUNT_SW_DUMMY as u64, + sample_type: (PERF_SAMPLE_TID | PERF_SAMPLE_TIME) as u64, + // PERF_FORMAT_LOST cannot account for inherited child events from this + // parent fd, so PERF_RECORD_LOST remains the complete loss signal. + read_format: 0, + clockid: libc::CLOCK_MONOTONIC, + ..Default::default() + }; + attr.__bindgen_anon_2.wakeup_events = 1; + attr.set_disabled(1); + attr.set_inherit(1); + attr.set_mmap(1); + attr.set_sample_id_all(1); + attr.set_mmap2(1); + attr.set_use_clockid(1); + + let fd = unsafe { + perf_event_open_sys::perf_event_open( + &mut attr, + pid, + cpu as _, + -1, + PERF_FLAG_FD_CLOEXEC as _, + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()) + .with_context(|| format!("perf_event_open failed for pid {pid} on CPU {cpu}")); + } + + let mapping = unsafe { + libc::mmap( + ptr::null_mut(), + mapping_len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if mapping == libc::MAP_FAILED { + let error = io::Error::last_os_error(); + unsafe { libc::close(fd) }; + return Err(error).context("failed to mmap perf mapping-event ring buffer"); + } + + let page = unsafe { &*(mapping.cast::()) }; + let data_offset = match usize::try_from(page.data_offset) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data offset"); + } + }; + let data_size = match usize::try_from(page.data_size) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data size"); + } + }; + let ring = Self { + fd, + mapping: mapping.cast(), + mapping_len, + data_offset, + data_size, + enabled: false, + }; + + ensure!( + data_offset >= page_size && data_offset % page_size == 0, + "kernel returned an invalid perf ring data offset" + ); + ensure!( + data_size >= size_of::() + && data_size % page_size == 0 + && data_size.is_power_of_two(), + "kernel returned an invalid perf ring data size" + ); + let data_end = data_offset + .checked_add(data_size) + .context("perf ring data range overflow")?; + ensure!( + data_end <= mapping_len, + "kernel returned a perf ring outside the mapped area" + ); + + Ok(ring) + } + + fn enable(&mut self) -> Result<()> { + if unsafe { perf_event_open_sys::ioctls::ENABLE(self.fd, 0) } < 0 { + return Err(io::Error::last_os_error()).context("failed to enable perf mapping events"); + } + self.enabled = true; + Ok(()) + } + + fn drain(&mut self, mappings: &mut Vec, lost: &AtomicU64) { + let page = unsafe { &mut *(self.mapping.cast::()) }; + let head = unsafe { ptr::read_volatile(&page.data_head) }; + std::sync::atomic::fence(Ordering::Acquire); + let mut tail = unsafe { ptr::read_volatile(&page.data_tail) }; + let available = head.wrapping_sub(tail); + + // Once the producer has lapped the consumer, the beginning of the + // stream no longer has a record boundary. Skip the corrupt prefix and + // let the kernel's PERF_RECORD_LOST record account for normal overflow. + if available > self.data_size as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + } else { + while tail != head { + let available = head.wrapping_sub(tail); + if available < size_of::() as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let header = self.copy_from_ring(tail, size_of::()); + let size = u16::from_ne_bytes([header[6], header[7]]) as usize; + if !(size_of::()..=self.data_size).contains(&size) + || size as u64 > available + { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let record = self.copy_from_ring(tail, size); + self.handle_record(&record, mappings, lost); + tail = tail.wrapping_add(size as u64); + } + } + + std::sync::atomic::fence(Ordering::Release); + unsafe { ptr::write_volatile(&mut page.data_tail, tail) }; + } + + fn copy_from_ring(&self, offset: u64, len: usize) -> Vec { + debug_assert!(len <= self.data_size); + let start = offset as usize & (self.data_size - 1); + let first_len = len.min(self.data_size - start); + let data = unsafe { self.mapping.add(self.data_offset) }; + let mut out = Vec::with_capacity(len); + unsafe { + out.extend_from_slice(std::slice::from_raw_parts(data.add(start), first_len)); + if first_len < len { + out.extend_from_slice(std::slice::from_raw_parts(data, len - first_len)); + } + } + out + } + + fn handle_record(&self, record: &[u8], mappings: &mut Vec, lost: &AtomicU64) { + match read_u32(record, 0) { + Some(PERF_RECORD_MMAP2) => { + if let Some(event) = parse_mmap2(record) { + mappings.push(event); + } + } + Some(PERF_RECORD_LOST) => match read_u64(record, 16) { + Some(count) => { + lost.fetch_add(count, Ordering::Relaxed); + } + None => { + lost.fetch_add(1, Ordering::Relaxed); + } + }, + _ => {} + } + } +} + +impl Drop for PerfRing { + fn drop(&mut self) { + unsafe { + if self.enabled { + let _ = perf_event_open_sys::ioctls::DISABLE(self.fd, 0); + } + libc::munmap(self.mapping.cast(), self.mapping_len); + libc::close(self.fd); + } + } +} + +pub(crate) struct PerfMappingPoller { + ctl: Option>>, + thread: Option>, +} + +impl PerfMappingPoller { + pub(crate) fn start( + pid: libc::pid_t, + tx: Sender, + lost: Arc, + ) -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "failed to read the system page size"); + + let cpus = online_cpus()?; + ensure!(!cpus.is_empty(), "no online CPUs reported by the kernel"); + let mut rings = Vec::with_capacity(cpus.len()); + for cpu in cpus { + rings.push(PerfRing::open(pid, cpu, page_size as usize)?); + } + for ring in &mut rings { + ring.enable()?; + } + + let (ctl, ctl_rx) = mpsc::channel::>(); + let thread = std::thread::spawn(move || { + let mut mappings = Vec::new(); + loop { + match ctl_rx.recv_timeout(Duration::from_millis(10)) { + Ok(ack) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + let _ = ack.send(()); + } + Err(RecvTimeoutError::Timeout) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + } + Err(RecvTimeoutError::Disconnected) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + mappings.sort_unstable_by_key(|event| (event.pid, event.timestamp)); + for mapping in mappings { + let _ = tx.send(mapping); + } + break; + } + } + } + }); + + Ok(Self { + ctl: Some(ctl), + thread: Some(thread), + }) + } +} + +impl Drop for PerfMappingPoller { + fn drop(&mut self) { + drop(self.ctl.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn parse_mmap2(record: &[u8]) -> Option { + const FIXED_END: usize = 72; + const SAMPLE_ID_SIZE: usize = 16; + if record.len() < FIXED_END + SAMPLE_ID_SIZE + || read_u32(record, 0)? != PERF_RECORD_MMAP2 + || read_u16(record, 6)? as usize != record.len() + { + return None; + } + + let prot = read_u32(record, 64)?; + if prot & libc::PROT_EXEC as u32 == 0 { + return None; + } + + let path_end = record.len() - SAMPLE_ID_SIZE; + let path_bytes = &record[FIXED_END..path_end]; + let nul = path_bytes.iter().position(|byte| *byte == 0)?; + let path = std::str::from_utf8(&path_bytes[..nul]).ok()?; + if !path.starts_with('/') { + return None; + } + + let major = read_u32(record, 40)? as u64; + let minor = read_u32(record, 44)? as u64; + Some(MemtrackEvent { + pid: read_u32(record, 8)? as libc::pid_t, + tid: read_u32(record, 12)? as libc::pid_t, + timestamp: read_u64(record, record.len() - 8)?, + addr: read_u64(record, 16)?, + kind: MemtrackEventKind::Mapping { + path: path.to_owned(), + dev: (major << 20) | minor, + ino: read_u64(record, 48)?, + file_offset: read_u64(record, 32)?, + len: read_u64(record, 24)?, + }, + }) +} + +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_ne_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_ne_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_ne_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +fn online_cpus() -> Result> { + let spec = std::fs::read_to_string("/sys/devices/system/cpu/online") + .context("failed to read online CPUs")?; + parse_cpu_list(spec.trim()) +} + +fn parse_cpu_list(spec: &str) -> Result> { + let mut cpus = Vec::new(); + for part in spec.split(',') { + let part = part.trim(); + ensure!(!part.is_empty(), "invalid empty CPU range"); + let (start, end) = match part.split_once('-') { + Some((start, end)) => (start.parse::()?, end.parse::()?), + None => { + let cpu = part.parse::()?; + (cpu, cpu) + } + }; + ensure!(start <= end, "invalid CPU range {part}"); + cpus.extend(start..=end); + } + Ok(cpus) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_cpu_ranges() { + assert_eq!(parse_cpu_list("0-2,5,8-9").unwrap(), vec![0, 1, 2, 5, 8, 9]); + } + + #[test] + fn parses_executable_mmap2() { + let path = b"/tmp/module.so\0"; + let mut record = vec![0; 72 + path.len() + 16]; + record[0..4].copy_from_slice(&PERF_RECORD_MMAP2.to_ne_bytes()); + let size = record.len() as u16; + record[6..8].copy_from_slice(&size.to_ne_bytes()); + record[8..12].copy_from_slice(&7_u32.to_ne_bytes()); + record[12..16].copy_from_slice(&8_u32.to_ne_bytes()); + record[16..24].copy_from_slice(&0x4000_u64.to_ne_bytes()); + record[24..32].copy_from_slice(&0x2000_u64.to_ne_bytes()); + record[32..40].copy_from_slice(&0x1000_u64.to_ne_bytes()); + record[40..44].copy_from_slice(&1_u32.to_ne_bytes()); + record[44..48].copy_from_slice(&2_u32.to_ne_bytes()); + record[48..56].copy_from_slice(&42_u64.to_ne_bytes()); + record[64..68].copy_from_slice(&(libc::PROT_EXEC as u32).to_ne_bytes()); + record[72..72 + path.len()].copy_from_slice(path); + let timestamp = 99_u64; + let time_offset = record.len() - 8; + record[time_offset..].copy_from_slice(×tamp.to_ne_bytes()); + + assert_eq!( + parse_mmap2(&record), + Some(MemtrackEvent { + pid: 7, + tid: 8, + timestamp, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: "/tmp/module.so".into(), + dev: (1 << 20) | 2, + ino: 42, + file_offset: 0x1000, + len: 0x2000, + }, + }) + ); + } +} diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 551db639c..c8f7ef475 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -1,4 +1,5 @@ use crate::ebpf::poller::RingBufferPoller; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; use std::process::{Child, ExitStatus}; @@ -9,9 +10,15 @@ use std::sync::mpsc::Receiver; pub struct Session { child: Child, events: Option>, + + // Drop order is part of the artifact compatibility contract. Rust drops + // fields in declaration order: both BPF pollers must stay before the perf + // mapping poller. Their Drop implementations disconnect, fully drain, and + // join their poll threads before PerfMappingPoller drops and emits its + // buffered Mapping records as the terminal stream suffix. _poller: RingBufferPoller, _stack_poller: Option, - _mapping_poller: Option, + _perf_mapping_poller: Option, } impl Session { @@ -20,14 +27,14 @@ impl Session { events: Receiver, poller: RingBufferPoller, stack_poller: Option, - mapping_poller: Option, + perf_mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, _stack_poller: stack_poller, - _mapping_poller: mapping_poller, + _perf_mapping_poller: perf_mapping_poller, } } diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index d217ff392..f74de0530 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -2,14 +2,10 @@ mod shared; use rstest::rstest; +use shared::AllocationTestCase; use std::process::Command; use tempfile::TempDir; -struct AllocationTestCase { - name: &'static str, - source: &'static str, -} - const ALLOCATION_TEST_CASES: &[AllocationTestCase] = &[ AllocationTestCase { name: "double_malloc", diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 9bc2e2e9b..e0b65afe6 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -9,6 +9,11 @@ use std::process::Command; pub type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; +pub struct AllocationTestCase { + pub name: &'static str, + pub source: &'static str, +} + /// Snapshot every tracked event, ordered by timestamp and deduplicated by /// `(addr, kind)` so repeated tracking of one allocation counts once. /// @@ -177,7 +182,11 @@ pub fn compile_rust_binary( /// Track a binary, collecting all memory events. pub fn track_binary(binary: &Path) -> TrackResult { - track_command(Command::new(binary)) + track_command(Command::new(binary), None) +} + +pub fn track_binary_with_env(binary: &Path) -> TrackResult { + track_command_with_tracker(Command::new(binary), Tracker::new()?) } pub fn compile_c_source( @@ -201,15 +210,10 @@ pub fn compile_c_source( Ok(binary_path) } -/// Track a command with the default probes: no rmap, and allocators discovered -/// by the exec-mapping watcher as the tracked tree maps executables. -pub fn track_command(command: Command) -> TrackResult { - track_command_with_opts(command, TrackerOptions::builder().build()) -} - -/// Track a command under a specific BPF variant rather than the detected one. -pub fn track_command_with_variant(command: Command, variant: BpfVariant) -> TrackResult { - track_command_with_tracker(command, Tracker::with_variant(variant)?) +/// Track a command with explicit options, using builder defaults when absent. +pub fn track_command(command: Command, options: impl Into>) -> TrackResult { + let tracker = Tracker::with_options(options.into().unwrap_or_default())?; + track_command_with_tracker(command, tracker) } /// RSS reconstruction from the folio rmap hooks, without allocator probes. @@ -220,16 +224,6 @@ fn rmap_only_options() -> TrackerOptions { .build() } -/// Track a command with folio rmap hooks enabled, reconstructing per-process RSS. -pub fn track_command_with_rmap(command: Command) -> TrackResult { - track_command_with_opts(command, rmap_only_options()) -} - -/// Track a command with an explicit probe selection rather than the environment's. -pub fn track_command_with_opts(command: Command, options: TrackerOptions) -> TrackResult { - track_command_with_tracker(command, Tracker::with_options(options)?) -} - /// Track a command with rmap hooks and snapshot its ownership maps after the /// tracked tree exits but before tracker teardown frees the BPF maps. pub fn track_command_with_rmap_maps( @@ -241,14 +235,6 @@ pub fn track_command_with_rmap_maps( Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } -/// Track a command with allocation stack capture enabled, returning its events. -pub fn track_command_with_stacks(command: Command) -> TrackResult { - track_command_with_opts( - command, - TrackerOptions::builder().stack_capture(true).build(), - ) -} - /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. /// @@ -319,7 +305,8 @@ pub fn for_each_variant( let mut profiles: Vec<(BpfVariant, EventProfile)> = Vec::new(); for variant in [BpfVariant::Legacy, BpfVariant::Token] { - let tracker = match Tracker::with_variant(variant) { + let options = TrackerOptions::builder().variant(Some(variant)).build(); + let tracker = match Tracker::with_options(options) { Ok(tracker) => tracker, Err(err) => { eprintln!("skipping {variant:?} variant, cannot attach here: {err:#}"); diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap new file mode 100644 index 000000000..2bd6a37ce --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: false }", + "Malloc { size: 2048, has_stack: false }", + "Malloc { size: 4096, has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap new file mode 100644 index 000000000..2bd6a37ce --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: false }", + "Malloc { size: 2048, has_stack: false }", + "Malloc { size: 4096, has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths_stack_capture_disabled.snap similarity index 100% rename from crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap rename to crates/memtrack/tests/snapshots/stack_tests__stack_paths_stack_capture_disabled.snap diff --git a/crates/memtrack/tests/stack_budget_tests.rs b/crates/memtrack/tests/stack_budget_tests.rs new file mode 100644 index 000000000..d5f80ba9a --- /dev/null +++ b/crates/memtrack/tests/stack_budget_tests.rs @@ -0,0 +1,22 @@ +//! The stack copy budget is a frozen rodata constant, so the verifier's cost of +//! the capture program scales with it. Loading at the default proves nothing +//! about the maximum; both must load. +use memtrack::{BpfVariant, MemtrackBpf, TrackerOptions}; +use rstest::rstest; + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(8192)] +#[case(u32::MAX)] +#[test_log::test] +fn skeleton_loads_at_stack_budget(#[case] budget: u32) { + for variant in [BpfVariant::Legacy, BpfVariant::Token] { + let options = TrackerOptions::builder() + .variant(Some(variant)) + .stack_budget(budget) + .build(); + MemtrackBpf::new(&options).unwrap_or_else(|e| { + panic!("{variant:?} skeleton failed to load at budget {budget}: {e:#}") + }); + } +} diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 28950dd72..bc6ee76b3 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -1,148 +1,108 @@ #[macro_use] mod shared; +use itertools::Itertools; +use memtrack::TrackerOptions; +use rstest::rstest; use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; -use std::collections::HashSet; +use shared::AllocationTestCase; +use std::mem::discriminant; use std::process::Command; use tempfile::TempDir; -const COPY_SIZE: u32 = 8192; - -fn compile_fixture( - name: &str, - temp_dir: &TempDir, -) -> Result> { - shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - name, - temp_dir.path(), - ) -} -fn require_mapping_support() -> bool { - if memtrack::MappingSupport::detect() == memtrack::MappingSupport::Unsupported { - eprintln!("skipping stack capture test: mapping support is unavailable"); - return false; - } - true -} - -/// The stack identity carried by each allocation and deallocation event that has one. -fn event_hashes(events: &[MemtrackEvent]) -> Vec { - events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } - | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), - _ => None, - }) - .collect() -} +fn describe_allocator_event(kind: &MemtrackEventKind) -> Option { + let description = match kind { + MemtrackEventKind::Malloc { size, stack_hash } => { + format!("Malloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::Calloc { size, stack_hash } => { + format!("Calloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::AlignedAlloc { size, stack_hash } => format!( + "AlignedAlloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ), + MemtrackEventKind::Realloc { + size, stack_hash, .. + } => { + format!( + "Realloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ) + } + MemtrackEventKind::Free { stack_hash } => { + format!("Free {{ has_stack: {} }}", *stack_hash != 0) + } + _ => return None, + }; -fn record_hashes(events: &[MemtrackEvent]) -> HashSet { - events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record } => Some(record.hash), - _ => None, - }) - .collect() + Some(description) } -#[test_with::env(GITHUB_ACTIONS)] -#[test_log::test] -fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths", &temp_dir)?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; - - let records: Vec<_> = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record: r } => { - Some((r.hash, r.sp, &r.regs, &r.bytes, r.truncated)) - } - _ => None, - }) - .collect(); - - assert!( - records.len() >= 2, - "expected at least two stack records, got {} ({} events)", - records.len(), - events.len() - ); - - let hashes = record_hashes(&events); - assert_eq!( - hashes.len(), - records.len(), - "stack records must be deduplicated by unique hash" - ); - - for (hash, sp, regs, bytes, truncated) in &records { - assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); - assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); - assert!( - !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, - "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", - bytes.len() - ); - assert_eq!( - *truncated, - bytes.len() == COPY_SIZE as usize, - "record {hash:#x} may only be flagged truncated when it filled the budget" - ); - } - - let carried = event_hashes(&events); - assert!( - !carried.is_empty(), - "expected events carrying a captured stack hash" - ); - assert!( - carried.iter().all(|hash| hashes.contains(hash)), - "every non-zero stack_hash must have a matching stack record" - ); - - // The fixture frees every allocation, so both sides must report identities. - assert!( +fn format_events(events: &[MemtrackEvent]) -> Vec { + const MARKER: u64 = 0xC0D5_9EED; + let has_markers = events.iter().any(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { size, .. } if size == MARKER + ) + }); + + let filtered_events = if has_markers { + shared::between_markers(events) + } else { events .iter() - .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), - "free events must carry their own stack identity" - ); + .filter(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { .. } + | MemtrackEventKind::Free { .. } + | MemtrackEventKind::Calloc { .. } + | MemtrackEventKind::Realloc { .. } + | MemtrackEventKind::AlignedAlloc { .. } + ) + }) + .sorted_by_key(|e| e.timestamp) + .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) + .cloned() + .collect() + }; - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) + filtered_events + .iter() + .filter_map(|e| describe_allocator_event(&e.kind)) + .collect() } -#[test_with::env(GITHUB_ACTIONS)] -#[test_log::test] -fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } +const STACK_TEST_CASES: &[AllocationTestCase] = &[ + AllocationTestCase { + name: "stack_paths", + source: include_str!("../testdata/stack_paths.c"), + }, + AllocationTestCase { + name: "nested_doubling", + source: include_str!("../testdata/nested_doubling.c"), + }, + AllocationTestCase { + name: "nested_doubling_shared_free", + source: include_str!("../testdata/nested_doubling_shared_free.c"), + }, +]; + +fn assert_stack_snapshot( + test_case: &AllocationTestCase, + stack_capture: bool, + snapshot_name: &str, +) -> Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; + let binary = shared::compile_c_source(test_case.source, test_case.name, temp_dir.path())?; + let options = TrackerOptions::builder() + .stack_capture(stack_capture) + .build(); + let (events, thread_handle) = shared::track_command(Command::new(binary), options)?; - let carried = event_hashes(&events); - let records = record_hashes(&events); - assert!( - carried.len() > records.len(), - "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", - carried.len(), - records.len(), - events.len() - ); + insta::assert_debug_snapshot!(snapshot_name, format_events(&events)); thread_handle .join() @@ -150,134 +110,30 @@ fn dedup_collapses_repeated_call_paths() -> Result<(), Box Self { - // SAFETY: tests run with --test-threads 1, so no concurrent env access. - unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; - Self - } -} - -impl Drop for DisableCaptureGuard { - fn drop(&mut self) { - // SAFETY: see `set`. - unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; - } -} - #[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(&STACK_TEST_CASES[0])] +#[case(&STACK_TEST_CASES[1])] +#[case(&STACK_TEST_CASES[2])] #[test_log::test] -fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; - let _guard = DisableCaptureGuard::set(); - let (events, thread_handle) = shared::track_binary(&binary)?; - - assert!( - events - .iter() - .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), - "disabled capture must still report allocation events" - ); - assert!( - record_hashes(&events).is_empty(), - "disabled capture must emit zero stack records" - ); - assert!( - event_hashes(&events).is_empty(), - "disabled capture must leave stack_hash zero on every event" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) +fn test_stack_capture( + #[case] test_case: &AllocationTestCase, +) -> Result<(), Box> { + assert_stack_snapshot(test_case, true, test_case.name) } -/// The doubling fixtures allocate down a three-level call chain and free in the -/// reverse order, from three distinct depths (`nested_doubling.c`) or from the -/// innermost frame (`nested_doubling_shared_free.c`). Both must pair every free -/// with its allocation in reverse order and give each allocation depth its own -/// stack identity. #[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(&STACK_TEST_CASES[0])] +#[case(&STACK_TEST_CASES[1])] +#[case(&STACK_TEST_CASES[2])] #[test_log::test] -fn nested_doubling_frees_in_reverse_order() -> Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } - for (name, source) in [ - ( - "nested_doubling", - include_str!("../testdata/nested_doubling.c"), - ), - ( - "nested_doubling_shared_free", - include_str!("../testdata/nested_doubling_shared_free.c"), - ), - ] { - let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source(source, name, temp_dir.path())?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; - - let allocations: Vec<(u64, u64, u64)> = events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size, stack_hash } if (1024..=4096).contains(&size) => { - Some((size, e.addr, stack_hash)) - } - _ => None, - }) - .collect(); - - assert_eq!( - allocations - .iter() - .map(|(size, ..)| *size) - .collect::>(), - vec![1024, 2048, 4096], - "[{name}] each level must allocate twice its caller, outermost first" - ); - - // Both probes of one free report the same address, so dedup by address - // to recover the order the fixture released its buffers in. - let mut released: Vec = Vec::new(); - let mut free_hashes: Vec = Vec::new(); - for event in &events { - let MemtrackEventKind::Free { stack_hash } = event.kind else { - continue; - }; - if released.last() == Some(&event.addr) { - continue; - } - released.push(event.addr); - free_hashes.push(stack_hash); - } - - let allocated: Vec = allocations.iter().map(|(_, addr, _)| *addr).collect(); - let expected: Vec = allocated.iter().rev().copied().collect(); - assert_eq!( - released, expected, - "[{name}] buffers must be freed in the reverse of their allocation order" - ); - - let alloc_hashes: HashSet = allocations.iter().map(|(.., hash)| *hash).collect(); - assert_eq!( - alloc_hashes.len(), - allocations.len(), - "[{name}] each allocation depth must carry its own stack identity" - ); - assert!( - !alloc_hashes.contains(&0) && !free_hashes.contains(&0), - "[{name}] every allocation and free must carry a captured stack" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - } - Ok(()) +fn test_stack_capture_disabled( + #[case] test_case: &AllocationTestCase, +) -> Result<(), Box> { + assert_stack_snapshot( + test_case, + false, + &format!("{}_stack_capture_disabled", test_case.name), + ) } diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs deleted file mode 100644 index b271dfee4..000000000 --- a/crates/runner-shared/src/artifacts/memtrack/mappings.rs +++ /dev/null @@ -1,37 +0,0 @@ -use libc::pid_t; -use serde::{Deserialize, Serialize}; -use std::ops::Range; - -/// The file-backed mappings the tracked process tree loaded, recorded as they -/// happened. Companion to the event stream: allocation stacks are raw -/// addresses, and these are what turns them back into modules. -/// -/// Kept out of the event stream so a consumer that only needs the module set -/// does not have to decode millions of allocation events. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MemtrackMappings { - pub mappings: Vec, -} - -impl super::super::ArtifactExt for MemtrackMappings {} - -/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` -/// would describe it. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessMapping { - pub pid: pid_t, - /// Resolved in-kernel at mmap time, so it is correct for the mapping - /// process's mount namespace even if the process is already gone. - pub path: String, - /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at - /// analysis time that the path still names the file that was mapped. - pub dev: u64, - pub ino: u64, - /// Offset of the mapping's first byte in the file. In bytes, matching - /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. - pub file_offset: u64, - pub avma_range: Range, - /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The - /// mapping is valid from here until a later mapping covers the range. - pub timestamp: u64, -} diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 0ba0ced47..fed115f29 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -2,11 +2,9 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::io::{BufReader, Read, Write}; -mod mappings; mod pipeline; mod writer; -pub use mappings::*; pub use pipeline::*; pub use writer::*; @@ -104,8 +102,18 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + /// One executable file mapping from a native PERF_RECORD_MMAP2 record. + /// The common event header carries its address, process, and timestamp. + Mapping { + path: String, + dev: u64, + ino: u64, + file_offset: u64, + len: u64, + }, Stack { + // Box keeps the MemtrackEventKind enum small across millions of events. #[serde(flatten)] record: Box, }, @@ -180,6 +188,19 @@ mod tests { size: 40960, }, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 400, + addr: 0x400000, + kind: MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, + }, ]; let artifact = MemtrackArtifact { @@ -239,6 +260,13 @@ mod tests { MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, MemtrackEventKind::Stack { record: Box::new(StackRecord { hash: 0xDEAD_BEEF, diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs index f8dcd0e9c..e888668bb 100644 --- a/src/executor/memory/module_artifacts.rs +++ b/src/executor/memory/module_artifacts.rs @@ -3,13 +3,35 @@ use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; -use runner_shared::artifacts::{ArtifactExt, MemtrackMappings, ProcessMapping}; +use libc::pid_t; +use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind}; use runner_shared::metadata::MemtrackMetadata; +use runner_shared::unwind_data::ProcessUnwindData; use std::collections::HashMap; +use std::ops::Range; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProcessMapping { + pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + dev: u64, + ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + file_offset: u64, + avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + timestamp: u64, +} /// Turn the mappings memtrack recorded into the artifacts an offline unwinder /// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the @@ -36,18 +58,13 @@ pub fn save_module_artifacts( ); let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); - MemtrackMetadata { - version: MEMTRACK_METADATA_CURRENT_VERSION, - integration, - artifacts: saved.artifacts, - } - .save_to(profile_folder) + MemtrackMetadata::new(integration, saved.artifacts).save_to(profile_folder) } /// Read every mapping artifact in the folder. One is written per tracked root /// process, so a run with several of them contributes several files. fn read_mappings(results_folder: &Path) -> Result> { - let suffix = format!(".{}.msgpack", MemtrackMappings::name()); + let suffix = format!(".{}.msgpack", MemtrackArtifact::name()); let mut mappings = Vec::new(); for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { @@ -56,10 +73,93 @@ fn read_mappings(results_folder: &Path) -> Result> { } let file = std::fs::File::open(entry.path())?; - let artifact = MemtrackMappings::decode_from_reader(file) - .with_context(|| format!("Failed to decode {:?}", entry.path()))?; - mappings.extend(artifact.mappings); + mappings.extend( + read_mappings_from_artifact(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?, + ); + } + + mappings.sort_unstable_by_key(|mapping| (mapping.pid, mapping.timestamp)); + Ok(mappings) +} + +/// Reconstruct mappings across forks because inherited perf events do not +/// synthesize mappings that already existed when a child was forked. +fn read_mappings_from_artifact(reader: R) -> Result> { + let mut timeline = MemtrackArtifact::decode_streamed(reader)? + .filter(|event| { + matches!( + &event.kind, + MemtrackEventKind::Exec + | MemtrackEventKind::Mapping { .. } + | MemtrackEventKind::Fork { .. } + ) + }) + .collect::>(); + + // Ties break so exec purges before mapping, while fork inherits that mapping. + timeline.sort_by_key(|event| { + let rank = match &event.kind { + MemtrackEventKind::Exec => 0, + MemtrackEventKind::Mapping { .. } => 1, + MemtrackEventKind::Fork { .. } => 2, + _ => unreachable!(), + }; + (event.timestamp, rank) + }); + + let mut live_mappings: HashMap> = HashMap::new(); + let mut mappings = Vec::new(); + + for event in timeline { + match event.kind { + MemtrackEventKind::Mapping { + path, + dev, + ino, + file_offset, + len, + } => { + let Some(end) = event.addr.checked_add(len) else { + debug!("Skipping mapping for {path}: address range overflows"); + continue; + }; + + let mapping = ProcessMapping { + pid: event.pid, + path, + dev, + ino, + file_offset, + avma_range: event.addr..end, + timestamp: event.timestamp, + }; + live_mappings + .entry(mapping.pid) + .or_default() + .push(mapping.clone()); + mappings.push(mapping); + } + MemtrackEventKind::Fork { parent_pid } => { + let inherited = live_mappings.get(&parent_pid).cloned().unwrap_or_default(); + let child_mappings = inherited + .into_iter() + .map(|mut mapping| { + mapping.pid = event.pid; + mapping.timestamp = event.timestamp; + mapping + }) + .collect::>(); + mappings.extend(child_mappings.iter().cloned()); + live_mappings.insert(event.pid, child_mappings); + } + MemtrackEventKind::Exec => { + live_mappings.remove(&event.pid); + } + _ => unreachable!(), + } } + Ok(mappings) } @@ -94,22 +194,29 @@ fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { - process_unwind_data.timestamp = Some(mapping.timestamp); - Some((unwind_data, process_unwind_data)) - } - Err(e) => { - debug!("Failed to load unwind data for {}: {e}", mapping.path); - None + let process_unwind_data = if let Some(unwind_data) = &loaded_module.unwind_data { + Some(ProcessUnwindData { + timestamp: Some(mapping.timestamp), + avma_range: mapping.avma_range.clone(), + base_avma: unwind_data.base_svma.wrapping_add(load_bias), + }) + } else { + match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + loaded_module.unwind_data = Some(unwind_data); + process_unwind_data.timestamp = Some(mapping.timestamp); + Some(process_unwind_data) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } } }; @@ -119,8 +226,7 @@ fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap bool { #[cfg(all(test, target_os = "linux"))] mod tests { use super::*; + use runner_shared::artifacts::MemtrackEvent; fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { ProcessMapping { @@ -216,7 +323,166 @@ mod tests { } #[test] - fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { + fn sorts_interleaved_mapping_artifacts_by_pid_and_timestamp() { + let results = tempfile::tempdir().unwrap(); + + // Separate files model records drained from different per-CPU rings. + MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 20, + addr: 0x2000, + kind: MemtrackEventKind::Mapping { + path: "second-module.so".to_string(), + dev: 2, + ino: 2, + file_offset: 0x2000, + len: 0x1000, + }, + }, + MemtrackEvent { + pid: 7, + tid: 7, + timestamp: 30, + addr: 0x7000, + kind: MemtrackEventKind::Mapping { + path: "child-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0x3000, + len: 0x1000, + }, + }, + ], + } + .save_file_to(results.path(), "cpu1.MemtrackArtifact.msgpack") + .unwrap(); + MemtrackArtifact { + events: vec![MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 10, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: "first-module.so".to_string(), + dev: 1, + ino: 1, + file_offset: 0x1000, + len: 0x1000, + }, + }], + } + .save_file_to(results.path(), "cpu0.MemtrackArtifact.msgpack") + .unwrap(); + + let mappings = read_mappings(results.path()).unwrap(); + assert_eq!( + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.timestamp)) + .collect::>(), + vec![(7, 30), (42, 10), (42, 20)] + ); + assert_eq!(mappings[0].path, "child-module.so"); + assert_eq!(mappings[1].path, "first-module.so"); + assert_eq!(mappings[2].path, "second-module.so"); + } + #[test] + fn extracts_all_mapping_events_from_a_streamed_artifact() { + const FIRST_MODULE: &str = "first-module.so"; + const SECOND_MODULE: &str = "second-module.so"; + let artifact = MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 10, + addr: 0, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, + }, + MemtrackEvent { + pid: 7, + tid: 8, + timestamp: 11, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + len: 0x2000, + }, + }, + MemtrackEvent { + pid: 9, + tid: 10, + timestamp: 12, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + len: 0x3000, + }, + }, + MemtrackEvent { + pid: 99, + tid: 99, + timestamp: 99, + addr: u64::MAX, + kind: MemtrackEventKind::Mapping { + path: "overflow-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0, + len: 1, + }, + }, + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 13, + addr: 0x2000, + kind: MemtrackEventKind::Free { stack_hash: 0 }, + }, + ], + }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + + assert_eq!( + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap(), + vec![ + ProcessMapping { + pid: 7, + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + avma_range: 0x1000..0x3000, + timestamp: 11, + }, + ProcessMapping { + pid: 9, + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + avma_range: 0x4000..0x7000, + timestamp: 12, + }, + ] + ); + } + + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_streamed_mapping() { const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; let profile = tempfile::tempdir().unwrap(); @@ -224,15 +490,19 @@ mod tests { std::fs::create_dir_all(&results).unwrap(); let (dev, ino) = s_dev_of(MODULE); - MemtrackMappings { - mappings: vec![ProcessMapping { + MemtrackArtifact { + events: vec![MemtrackEvent { pid: 1234, - path: MODULE.to_string(), - dev, - ino, - file_offset: 0x5_2000, - avma_range: 0x5555_555a_7000..0x5555_556b_0000, + tid: 1234, timestamp: 999, + addr: 0x5555_555a_7000, + kind: MemtrackEventKind::Mapping { + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + len: 0x109_000, + }, }], } .save_with_pid_to(&results, 1234) @@ -250,7 +520,7 @@ mod tests { ) .unwrap(); - assert_eq!(metadata.version, MEMTRACK_METADATA_CURRENT_VERSION); + assert_eq!(metadata.version, MemtrackMetadata::CURRENT_VERSION); assert_eq!( metadata.artifacts.mapped_process_module_symbols[&1234].len(), 1 @@ -269,4 +539,133 @@ mod tests { PathBuf::from(MODULE) ); } + + fn mapping_event(pid: pid_t, timestamp: u64, addr: u64, path: &str) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr, + kind: MemtrackEventKind::Mapping { + path: path.to_string(), + dev: 1, + ino: 1, + file_offset: 0, + len: 0x1000, + }, + } + } + + fn fork_event(child_pid: pid_t, parent_pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid: child_pid, + tid: child_pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Fork { parent_pid }, + } + } + + fn exec_event(pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Exec, + } + } + + fn decode_lifecycle(events: Vec) -> Vec { + let artifact = MemtrackArtifact { events }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap() + } + + fn mapping_summary(mappings: &[ProcessMapping]) -> Vec<(pid_t, &str, u64)> { + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.path.as_str(), mapping.timestamp)) + .collect() + } + + #[test] + fn fork_without_exec_inherits_only_mappings_before_fork() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + mapping_event(100, 30, 0x2000, "after-fork.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 30), + ] + ); + } + + #[test] + fn exec_stops_inheriting_parent_mappings() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + exec_event(200, 30), + mapping_event(100, 40, 0x2000, "after-fork.so"), + mapping_event(200, 50, 0x3000, "after-exec.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 40), + (200, "after-exec.so", 50), + ] + ); + } + + #[test] + fn grandchild_inherits_transitively_from_forked_child() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "root.so"), + fork_event(200, 100, 20), + mapping_event(200, 25, 0x2000, "child.so"), + fork_event(300, 200, 30), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "root.so", 10), + (200, "root.so", 20), + (200, "child.so", 25), + (300, "root.so", 30), + (300, "child.so", 30), + ] + ); + } + + #[test] + fn equal_timestamp_events_follow_exec_mapping_fork_rank() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-exec.so"), + fork_event(200, 100, 20), + mapping_event(100, 20, 0x2000, "after-exec.so"), + exec_event(100, 20), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-exec.so", 10), + (100, "after-exec.so", 20), + (200, "after-exec.so", 20), + ] + ); + } } From 76e4291671c8f9db52da8351193a8ae8e2d029e5 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 19:43:01 +0200 Subject: [PATCH 13/18] perf(runner-shared): pre-size the frame output buffer Every frame's output buffer started empty and doubled its way to the compressed size, which for a 64k event frame is 8 realloc-and-copy steps over roughly 8 MB. Not measurable in wall clock at current frame sizes; it removes the copy traffic. --- crates/runner-shared/src/artifacts/memtrack/pipeline.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index 8cac46f05..5cc9a10e2 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -13,6 +13,10 @@ const FRAME_EVENTS: usize = 64 * 1024; /// memory to roughly `FRAME_EVENTS * WINDOW_FRAMES` events regardless of how long /// the source runs. const WINDOW_FRAMES: usize = 16; +/// Compressed bytes to reserve per event, so a frame's output buffer does not +/// grow-and-copy its way up from zero. Overshooting wastes a little memory per +/// in-flight frame; undershooting only costs the doublings it fails to avoid. +const FRAME_BYTES_PER_EVENT: usize = 16; /// Encode a stream of events into a single compressed artifact stream, /// compressing frames in parallel across a Rayon pool of `n_workers` threads. @@ -73,7 +77,7 @@ where /// Encode one batch as a single self-contained zstd frame. fn encode_frame(batch: &[MemtrackEvent]) -> anyhow::Result> { - let mut writer = MemtrackWriter::new(Vec::new())?; + let mut writer = MemtrackWriter::new(Vec::with_capacity(batch.len() * FRAME_BYTES_PER_EVENT))?; for event in batch { writer.write_event(event)?; } From 5edcca0b83bad6e2655d9b848d50a56e46f19190 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 19:43:10 +0200 Subject: [PATCH 14/18] ci: benchmark runner-shared in memory mode The memory instrument reports allocation counts and bytes per benchmark, which is what the encode path is bound by. It runs on a hosted runner like simulation does; the runner grants memtrack its capabilities during setup. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3265a9194..d2d363fb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: strategy: fail-fast: false matrix: - mode: [simulation, walltime] + mode: [simulation, walltime, memory] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: From 9086deda781554e83f261c9befd8f46fcaa49251 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 3 Sep 2026 12:24:33 +0200 Subject: [PATCH 15/18] refactor(runner-shared): inline pid-key deserializer into metadata --- crates/runner-shared/src/lib.rs | 1 - crates/runner-shared/src/metadata.rs | 32 ++++++++++++++++++-- crates/runner-shared/src/serde_pid_map.rs | 36 ----------------------- src/executor/helpers/debug_file.rs | 29 +++++++++++------- 4 files changed, 47 insertions(+), 51 deletions(-) delete mode 100644 crates/runner-shared/src/serde_pid_map.rs diff --git a/crates/runner-shared/src/lib.rs b/crates/runner-shared/src/lib.rs index 2cdc7d5a6..61e804de7 100644 --- a/crates/runner-shared/src/lib.rs +++ b/crates/runner-shared/src/lib.rs @@ -4,6 +4,5 @@ pub mod fifo; pub mod metadata; pub mod module_symbols; pub mod perf_event; -pub mod serde_pid_map; pub mod unwind_data; pub mod walltime_results; diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 2e48a2fd8..7f41a2088 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -1,5 +1,6 @@ use anyhow::Context; use libc::pid_t; +use serde::de::{Deserializer, Error}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::BufWriter; @@ -11,6 +12,31 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; +/// Reads a pid-keyed map whose keys arrive as strings. +/// +/// JSON object keys are always strings. serde_json's direct deserializer +/// special-cases that and parses integer map keys, but a `#[serde(flatten)]` +/// field is buffered into serde's internal `Content` first, and that path has +/// no such special case — a `pid_t` key then fails with `invalid type: string`. +/// See . +/// +/// Only the read side needs this: serializing writes the same bytes either way. +fn pid_keys_from_strings<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} + /// The per-profile module artifacts: the deduplicated debug info, unwind data /// and symbol tables extracted from the ELF modules the profiled processes /// mapped, plus the per-pid references into them. @@ -28,7 +54,7 @@ pub struct ModuleArtifacts { #[serde( default, skip_serializing_if = "HashMap::is_empty", - with = "crate::serde_pid_map" + deserialize_with = "pid_keys_from_strings" )] pub mapped_process_debug_info_by_pid: HashMap>, @@ -37,7 +63,7 @@ pub struct ModuleArtifacts { #[serde( default, skip_serializing_if = "HashMap::is_empty", - with = "crate::serde_pid_map" + deserialize_with = "pid_keys_from_strings" )] pub mapped_process_unwind_data_by_pid: HashMap>, @@ -46,7 +72,7 @@ pub struct ModuleArtifacts { #[serde( default, skip_serializing_if = "HashMap::is_empty", - with = "crate::serde_pid_map" + deserialize_with = "pid_keys_from_strings" )] pub mapped_process_module_symbols: HashMap>, diff --git a/crates/runner-shared/src/serde_pid_map.rs b/crates/runner-shared/src/serde_pid_map.rs deleted file mode 100644 index fa6fbb9dc..000000000 --- a/crates/runner-shared/src/serde_pid_map.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! `#[serde(with = ...)]` support for pid-keyed maps. -//! -//! JSON object keys are always strings. serde_json's direct deserializer -//! special-cases that and parses integer map keys, but a `#[serde(flatten)]` -//! field is buffered into serde's internal `Content` first, and that path has no -//! such special case — an `i32` key then fails with `invalid type: string`. So -//! the keys are read as strings and parsed here, which works on both paths. - -use libc::pid_t; -use serde::de::{Deserializer, Error}; -use serde::{Deserialize, Serialize, Serializer}; -use std::collections::HashMap; - -pub fn serialize(map: &HashMap, serializer: S) -> Result -where - V: Serialize, - S: Serializer, -{ - map.serialize(serializer) -} - -pub fn deserialize<'de, V, D>(deserializer: D) -> Result, D::Error> -where - V: Deserialize<'de>, - D: Deserializer<'de>, -{ - HashMap::::deserialize(deserializer)? - .into_iter() - .map(|(key, value)| { - let pid = key - .parse::() - .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; - Ok((pid, value)) - }) - .collect() -} diff --git a/src/executor/helpers/debug_file.rs b/src/executor/helpers/debug_file.rs index 0619bed2b..bb1c9d2ba 100644 --- a/src/executor/helpers/debug_file.rs +++ b/src/executor/helpers/debug_file.rs @@ -12,11 +12,17 @@ use std::path::{Path, PathBuf}; /// /// [Separate Debug Files]: https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html pub fn find_debug_file(object: &object::File, binary_path: &Path) -> Option { - ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] + let via_known_roots = ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] .iter() .map(Path::new) .filter(|dir| dir.exists()) - .find_map(|dir| find_debug_file_in(object, binary_path, dir)) + .find_map(|dir| find_debug_file_in(object, binary_path, dir)); + + // `.gnu_debuglink` beside the binary (and its `.debug` subdirectory) is part of + // GDB's search order regardless of whether a global debug-directory root exists + // (see module docs), so it must not be skipped on systems without one, e.g. most + // CI runners. + via_known_roots.or_else(|| find_debug_file_by_debuglink(object, binary_path, None)) } fn find_debug_file_in( @@ -27,7 +33,7 @@ fn find_debug_file_in( if let Some(path) = find_debug_file_by_build_id(object, debug_dir) { return Some(path); } - find_debug_file_by_debuglink(object, binary_path, debug_dir) + find_debug_file_by_debuglink(object, binary_path, Some(debug_dir)) } /// Build-id `a05cfb6313fe06a13c9b4b5cb86c2069faa3951f` resolves to @@ -58,19 +64,20 @@ fn find_debug_file_by_build_id(object: &object::File, debug_dir: &Path) -> Optio fn find_debug_file_by_debuglink( object: &object::File, binary_path: &Path, - debug_dir: &Path, + debug_dir: Option<&Path>, ) -> Option { let (debuglink, expected_crc) = object.gnu_debuglink().ok()??; let debuglink = std::str::from_utf8(debuglink).ok()?; let dir = binary_path.parent()?; - let candidates = [ - dir.join(debuglink), - dir.join(".debug").join(debuglink), - debug_dir - .join(dir.strip_prefix("/").unwrap_or(dir)) - .join(debuglink), - ]; + let mut candidates = vec![dir.join(debuglink), dir.join(".debug").join(debuglink)]; + if let Some(debug_dir) = debug_dir { + candidates.push( + debug_dir + .join(dir.strip_prefix("/").unwrap_or(dir)) + .join(debuglink), + ); + } candidates.into_iter().find(|p| { let Ok(content) = std::fs::read(p) else { From 0f9b9da22e882f8554ef736d51770c7c459ea37d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 3 Sep 2026 17:22:15 +0200 Subject: [PATCH 16/18] fix(memtrack): skip allocator symbols aliased at an attached offset glibc exports cfree at the same file offset as free, so attaching both instrumented one function twice: every free() produced two Free events and two stack captures. Track (library, offset) pairs and skip symbols already covered by an alias. --- crates/memtrack/src/ebpf/memtrack/macros.rs | 22 +++++++++++++++++++++ crates/memtrack/src/ebpf/memtrack/mod.rs | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index b1099abe0..dbfcaf276 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -85,8 +85,19 @@ macro_rules! attach_uprobe_uretprobe { let Some(offset) = symbols.offset(symbol) else { return Ok(false); }; + let key = (lib_path.to_path_buf(), offset); + if self.attached_offsets.contains(&key) { + log::trace!( + "Skipping alias {} at {:#x} in {} (already instrumented)", + symbol, + offset, + lib_path.display() + ); + return Ok(true); + } self.[](lib_path, offset) .with_context(|| format!("Failed to attach {symbol}"))?; + self.attached_offsets.insert(key); log::trace!("Attached {} at {:#x}", symbol, offset); Ok(true) } @@ -121,8 +132,19 @@ macro_rules! attach_uprobe { let Some(offset) = symbols.offset(symbol) else { return Ok(false); }; + let key = (lib_path.to_path_buf(), offset); + if self.attached_offsets.contains(&key) { + log::trace!( + "Skipping alias {} at {:#x} in {} (already instrumented)", + symbol, + offset, + lib_path.display() + ); + return Ok(true); + } self.[](lib_path, offset) .with_context(|| format!("Failed to attach {symbol}"))?; + self.attached_offsets.insert(key); log::trace!("Attached {} at {:#x}", symbol, offset); Ok(true) } diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 19cd898f3..523b275f3 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -120,6 +120,11 @@ pub struct MemtrackBpf { pub(super) skel: Skel, pub(super) probes: Vec, rmap: RmapSupport, + /// `(lib_path, offset)` pairs already instrumented. glibc exports + /// symbols like `cfree` (and `free_sized`/`__libc_free` elsewhere) at + /// the same file offset as their canonical function; attaching each + /// alias would double-instrument the one underlying function. + attached_offsets: std::collections::HashSet<(std::path::PathBuf, usize)>, } impl MemtrackBpf { @@ -224,6 +229,7 @@ impl MemtrackBpf { skel, probes: Vec::new(), rmap, + attached_offsets: std::collections::HashSet::new(), }) } From adddd9df73f6b115064dc60288f30c55681effd2 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 4 Sep 2026 10:56:01 +0200 Subject: [PATCH 17/18] perf(memtrack): switch to mimalloc to reduce memory usage and fragmentation mimalloc lowers memtrack's own memory usage, fragmentation and allocation overhead compared to glibc's allocator. As a side effect, it also doesn't route through the exported malloc/free/calloc/realloc symbols, so it skips the allocator uprobes (attached system-wide with pid -1) that would otherwise fire for memtrack's own bookkeeping allocations. Pulled in via the ebpf feature, which the binary already requires. --- Cargo.lock | 19 +++++++++++++++++++ crates/memtrack/Cargo.toml | 3 ++- crates/memtrack/src/main.rs | 3 +++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4956c2d3b..fee8f75f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2161,6 +2161,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "libredox" version = "0.1.16" @@ -2399,6 +2408,7 @@ dependencies = [ "libbpf-rs", "libc", "log", + "mimalloc", "object", "parking_lot", "paste", @@ -2438,6 +2448,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 68da1218f..82dccd265 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -16,7 +16,7 @@ required-features = ["ebpf"] [features] default = ["ebpf"] -ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux"] +ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux", "dep:mimalloc"] [dependencies] anyhow = { workspace = true } @@ -38,6 +38,7 @@ perf-event-open-sys = { workspace = true } rayon = "1.12" parking_lot = "0.12" typed-builder = "0.23.2" +mimalloc = { version = "0.1", optional = true } [build-dependencies] libbpf-cargo = { version = "0.26", optional = true } diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index d9a6411b1..64951cf2a 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -1,3 +1,6 @@ +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + use clap::Parser; use ipc_channel::ipc; use memtrack::prelude::*; From ed10235aee537954d1ac8e63df949d0837ea38fa Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 4 Sep 2026 10:56:15 +0200 Subject: [PATCH 18/18] perf(memtrack): resolve stack fp chains off the ring poll thread Each stack record needs its frame-pointer chain looked up in the stack_traces map, which is a syscall per record. Doing that inside the ring-buffer parse callback made the poll thread pay it, so a burst of stacks could push it behind the producer and records were dropped. ResolvingPoller wraps a RingBufferPoller with a dedicated resolver thread: the poll thread only parses (event, stackid) and hands it over an internal channel, and the resolver does the map lookup and forwards the completed event. Drop order keeps the existing shutdown contract, the ring is dropped first so its poll thread joins and closes the internal sender, which lets the resolver drain what it already has before its join returns. --- crates/memtrack/src/ebpf/memtrack/mod.rs | 32 +++++++------ crates/memtrack/src/ebpf/poller.rs | 60 ++++++++++++++++++++++++ crates/memtrack/src/session.rs | 6 +-- 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 523b275f3..9c8287763 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; -use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; use crate::ebpf::tracker::TrackerOptions; mod token { @@ -248,33 +248,37 @@ impl MemtrackBpf { )) } - /// Poll the stack-record ring buffer into `tx`. + /// Poll the stack-record ring buffer into `tx`. The map lookup for each + /// stack's frame-pointer chain is a syscall, so it runs on a resolver + /// thread rather than the ring poll thread: see [`ThreadedRingBufferPoller`]. pub(crate) fn poll_stacks( &self, poll_interval_ms: u64, tx: std::sync::mpsc::Sender, - ) -> Result { + ) -> Result { use crate::ebpf::events; use runner_shared::artifacts::MemtrackEventKind; - // The poller outlives this borrow of the skeleton, so the chain lookup - // needs an owned handle rather than a reference to the skeleton map. + // The resolver thread outlives this borrow of the skeleton, so the + // chain lookup needs an owned handle rather than a reference to the + // skeleton map. let stack_traces = with_skel!(self, skel => { libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) .context("Failed to create handle for stack_traces map")? }); - let parse = move |data: &[u8]| { - let (mut event, stackid) = events::parse_stack(data)?; - if let MemtrackEventKind::Stack { record } = &mut event.kind { - record.fp_chain = events::fp_chain(&stack_traces, stackid); - } - Some(event) - }; + let resolve = + move |(mut event, stackid): (runner_shared::artifacts::MemtrackEvent, i64)| { + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + event + }; - with_skel!(self, skel => RingBufferPoller::new( + with_skel!(self, skel => ThreadedRingBufferPoller::new( &skel.maps.stacks, - parse, + events::parse_stack, + resolve, tx, poll_interval_ms, )) diff --git a/crates/memtrack/src/ebpf/poller.rs b/crates/memtrack/src/ebpf/poller.rs index d80a15549..048fd30fe 100644 --- a/crates/memtrack/src/ebpf/poller.rs +++ b/crates/memtrack/src/ebpf/poller.rs @@ -77,3 +77,63 @@ impl Drop for RingBufferPoller { } } } + +/// A [`RingBufferPoller`] whose parsed items need a further, potentially +/// expensive step (e.g. a BPF map lookup, which is a syscall) before they are +/// forwarded on `tx`. That step runs on a dedicated resolver thread instead +/// of the poll thread, so a slow per-record resolve can't make the poll +/// thread fall behind the ring and drop records. +pub struct ThreadedRingBufferPoller { + // Declaration order is the shutdown order: dropping `ring` first + // disconnects its control channel and joins its poll thread, which drops + // the internal sender the resolver reads from. Only then can the + // resolver's `recv` loop observe disconnection, finish forwarding + // whatever it already has, and let `resolver`'s join below return. This + // gives callers the same "disconnect, fully drain, then join" contract + // as a plain `RingBufferPoller`. + ring: Option, + resolver: Option>, +} + +impl ThreadedRingBufferPoller { + /// Poll `rb_map` with `parse` like [`RingBufferPoller::new`], but run + /// `resolve` on a separate thread: `parse` results are forwarded over an + /// internal channel, and `resolve` turns each one into the value sent on + /// `tx`. + pub fn new( + rb_map: &M, + parse: F, + resolve: R, + tx: Sender, + poll_interval_ms: u64, + ) -> Result + where + M: MapCore, + T: Send + 'static, + U: Send + 'static, + F: Fn(&[u8]) -> Option + Send + 'static, + R: Fn(T) -> U + Send + 'static, + { + let (parsed_tx, parsed_rx) = mpsc::channel::(); + let ring = RingBufferPoller::new(rb_map, parse, parsed_tx, poll_interval_ms)?; + let resolver = std::thread::spawn(move || { + for item in parsed_rx { + let _ = tx.send(resolve(item)); + } + }); + + Ok(Self { + ring: Some(ring), + resolver: Some(resolver), + }) + } +} + +impl Drop for ThreadedRingBufferPoller { + fn drop(&mut self) { + drop(self.ring.take()); + if let Some(resolver) = self.resolver.take() { + let _ = resolver.join(); + } + } +} diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index c8f7ef475..53f02a50a 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -1,4 +1,4 @@ -use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; @@ -17,7 +17,7 @@ pub struct Session { // join their poll threads before PerfMappingPoller drops and emits its // buffered Mapping records as the terminal stream suffix. _poller: RingBufferPoller, - _stack_poller: Option, + _stack_poller: Option, _perf_mapping_poller: Option, } @@ -26,7 +26,7 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, - stack_poller: Option, + stack_poller: Option, perf_mapping_poller: Option, ) -> Self { Self {