From 3fc4cfa00b387fe970cde033a5acf57750053c45 Mon Sep 17 00:00:00 2001 From: Catalin Date: Sat, 10 Oct 2026 20:31:26 +0300 Subject: [PATCH 1/5] fix(dvfs): map M1 Max pmgr tables to hardware-validated buckets Classify GPU DVFS tables by shape rejected the real 7-entry GPU table (voltage-states9, 0-1296 MHz) and accepted the 15-entry P-core SRAM mirror (voltage-states5-sram, ceiling 3228 MHz): every GPU clock, P-state name, and derived TFLOPS figure on M1 Max was CPU telemetry wearing a GPU label. Add a brand-keyed override for Apple M1 Max and require any heuristic GPU candidate to clock strictly below the P-core max; unrecognized chips now degrade to the IOAccelerator fallback instead of publishing false clocks. --- actop/native_sys.py | 56 +++++++++++++++++++++++++++++++++++----- tests/test_native_sys.py | 51 ++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 7 deletions(-) diff --git a/actop/native_sys.py b/actop/native_sys.py index 393730c..08042c9 100644 --- a/actop/native_sys.py +++ b/actop/native_sys.py @@ -406,15 +406,56 @@ def get_dvfs_tables_native() -> dict: _cf.CFRelease(props_ref) + # SoC-specific pmgr key overrides win over the shape heuristic: on + # t8103 (M1 Max) the heuristic mis-assigns a *second P-core* table + # (voltage-states5-sram, 15 entries, 3228 MHz ceiling) as the GPU table + # because it is the first 10-20-entry table the CPU picks do not claim. + # The real GPU domain table is voltage-states9: [0, 388, 486, 648, 777, + # 972, 1296] — validated on hardware against powermetrics and the + # 10.4 TFLOPS @ 1293 MHz silicon spec (the GPU cannot clock at CPU rates). + brand = _get_cpu_brand() + override = _DVFS_TABLE_OVERRIDES.get(brand) + if override and all(k in tables for k in override.values()): + return {bucket: tables[key] for bucket, key in override.items()} + return _classify_dvfs_tables(tables) +# bucket -> pmgr voltage-states key, per SoC brand string. Only chips whose +# tables were read from that exact machine belong here; the heuristic below +# (and the api.py ioaccelerator fallback) covers everything else. +_DVFS_TABLE_OVERRIDES = { + "Apple M1 Max": { + "ecpu": "voltage-states1-sram", # [600, 972, 1332, 1704, 2064] + "pcpu": "voltage-states5-sram", # [600 ... 3228] + "gpu": "voltage-states9", # [0, 388 ... 1296] + }, +} + + +_CPU_BRAND = None + + +def _get_cpu_brand() -> str: + """machdep.cpu.brand_string, cached (immutable across uptime).""" + global _CPU_BRAND + if _CPU_BRAND is None: + _CPU_BRAND = get_sysctl_string("machdep.cpu.brand_string") or "" + return _CPU_BRAND + + def _classify_dvfs_tables(tables: dict) -> dict: """Classify raw voltage-states tables into ecpu/pcpu/gpu buckets. Heuristics: the P-core table has the highest max frequency and the most entries; the E-core table is small (5-12 entries); the GPU table has - 10-20 entries. Returns {'ecpu': [...], 'pcpu': [...], 'gpu': [...]}. + 10-20 entries AND a max strictly below the P-core ceiling (a GPU never + clocks at or above its chip's P-cluster max — the 15-16-entry P-core + SRAM mirror must not be mistaken for a GPU table). When no table passes, + 'gpu' stays empty and callers degrade honestly (util falls back to + IOAccelerator stats, frequency reports 0 / unavailable) rather than + publishing CPU clocks as GPU clocks. + Returns {'ecpu': [...], 'pcpu': [...], 'gpu': [...]}. """ ecpu = [] pcpu = [] @@ -441,15 +482,16 @@ def _classify_dvfs_tables(tables: dict) -> dict: ecpu = freqs break - # GPU: 10-20 entries, not pcpu or ecpu table, first match + # GPU: 10-20 entries, not pcpu or ecpu, ceiling strictly under P-core max for key, freqs in candidates: - if key == best_pcpu_key: + if key == best_pcpu_key or freqs is ecpu: continue - if freqs is ecpu: + if not (10 <= len(freqs) <= 20): continue - if 10 <= len(freqs) <= 20: - gpu = freqs - break + if not freqs or max(freqs) >= best_pcpu_max: + continue + gpu = freqs + break return {"ecpu": ecpu, "pcpu": pcpu, "gpu": gpu} diff --git a/tests/test_native_sys.py b/tests/test_native_sys.py index 1818177..eeb28ec 100644 --- a/tests/test_native_sys.py +++ b/tests/test_native_sys.py @@ -15,6 +15,7 @@ import pytest from actop.native_sys import ( + _classify_dvfs_tables, get_dvfs_tables_native, get_native_processes, get_thermal_pressure, @@ -144,3 +145,53 @@ def test_dvfs_tables_classify_into_plausible_buckets(): def test_thermal_pressure_returns_known_state(): assert get_thermal_pressure() in _THERMAL_STATES + + +# pmgr voltage-states captured from a live t8103 (M1 Max). The pre-1.9.13 +# shape heuristic mislabelled "voltage-states5-sram" — a second P-core SRAM +# table — as the GPU, because it was the first 10-20-entry table the CPU +# buckets had not claimed; every clock/TFLOPS reading derived from it was a +# CPU frequency wearing a GPU label. The 7-entry GPU table (voltage-states9, +# ceiling 1296 MHz) never matched the old 10-20-entry window at all. +_T8103_TABLES = { + "voltage-states1-sram": [600, 972, 1332, 1704, 2064], + "voltage-states5-sram": [ + 600, 828, 1056, 1296, 1524, 1752, 1980, 2208, + 2448, 2676, 2904, 3036, 3132, 3168, 3228, + ], + "voltage-states13-sram": [ + 600, 828, 1056, 1296, 1524, 1752, 1980, 2208, + 2448, 2676, 2904, 3036, 3132, 3168, 3228, + ], + "voltage-states8": [300, 600, 900, 1125, 1350, 1500], + "voltage-states9": [0, 389, 486, 648, 778, 972, 1296], + "voltage-states14": [0, 389, 486, 648, 778, 972, 1296], + "voltage-states31": [1000, 0, 0], +} + + +def test_classifier_never_hands_a_pcore_mirror_to_the_gpu_bucket(): + out = _classify_dvfs_tables(_T8103_TABLES) + assert out["pcpu"] == _T8103_TABLES["voltage-states5-sram"] + assert out["ecpu"] == _T8103_TABLES["voltage-states1-sram"] + # Either the GPU bucket stays empty (callers then degrade honestly: + # IOAccelerator utilization fallback, frequency n/a) or it holds a real + # accelerator table. A GPU clock at or above the P-core ceiling is + # physically impossible, so it is always a misclassification. + if out["gpu"]: + assert max(out["gpu"]) < max(out["pcpu"]) + assert len(out["gpu"]) != len(out["pcpu"]) + + +def test_m1_max_override_resolves_the_gpu_domain_table(): + import actop.native_sys as ns + + if ns._get_cpu_brand() != "Apple M1 Max": + pytest.skip("override table mapping validated against t8103 pmgr dumps") + tables = get_dvfs_tables_native() + assert tables["ecpu"] == [600, 972, 1332, 1704, 2064] + assert max(tables["pcpu"]) == 3228 + assert tables["gpu"], "M1 Max must resolve a GPU table via the override" + # 1296 MHz ceiling matches the M1 Max GPU spec (10.4 TFLOPS @ ~1293 MHz); + # tolerate ±50 MHz firmware drift but nothing CPU-shaped. + assert 1200 <= max(tables["gpu"]) <= 1350 From 5e27f36ca6306230bd6bc6884564d418fa56656e Mon Sep 17 00:00:00 2001 From: Catalin Date: Sat, 10 Oct 2026 20:32:18 +0300 Subject: [PATCH 2/5] fix(metrics): restore DRAM bandwidth, expose fabric demand, honest CPU rail The PMP filter demanded DCS BW channels starting with AMCC, but the DRAM-pin aggregate is named exactly RD+WR (AMCC totals live under AFR/AFI/AFC BW): bandwidth_available was false forever and the Mem BW row, chart, MEM-BOUND alert, and JSON field were all dead. Headline is now DCS RD+WR (GB/s at the pins); AFR AMCC RD+WR is exported as bandwidth_requested_gbps, and the TUI suffixes [req X] when fabric demand exceeds pinned traffic. _keep_states admits exactly the parsed channels, dropping per-agent histogram extraction that nothing reads. Unprivileged IOReport returns 0 mJ for every CPU energy channel on t8103 even at 8-core full load (verified against root powermetrics: 32.4 W), so a 0.00 W CPU reading was a fabricated measurement, not a gauge. The sampler now records whether any tick ever saw a nonzero CPU energy delta and exports it as cpu_power_available through models, api, JSON/Prometheus export, and the TUI, which renders the CPU row as 'n/a (needs root)' and annotates the package total '(GPU+ANE)' while the rail is blind. --- actop/api.py | 6 ++++ actop/export.py | 9 +++++ actop/models.py | 10 ++++++ actop/sampler.py | 78 +++++++++++++++++++++++++++++++++---------- actop/tui/app.py | 8 +++-- actop/tui/widgets.py | 26 +++++++++++++-- tests/test_sampler.py | 68 ++++++++++++++++++++++++++++++++++++- 7 files changed, 182 insertions(+), 23 deletions(-) diff --git a/actop/api.py b/actop/api.py index f2cacff..8faccc1 100644 --- a/actop/api.py +++ b/actop/api.py @@ -73,6 +73,7 @@ def _sample_to_snapshot( # total_gbps is a residency-weighted average already in GB/s — not a # byte counter, so it is not divided by the sample interval. total_bw = float(bw.get("total_gbps", 0.0)) if bw_avail else 0.0 + requested_bw = float(bw.get("requested_gbps", 0.0)) if bw_avail else 0.0 cpu_watts = cm["cpu_W"] / interval_s gpu_watts = cm["gpu_W"] / interval_s ane_watts = cm["ane_W"] / interval_s @@ -143,6 +144,11 @@ def _sample_to_snapshot( thermal_state=sample.thermal_pressure, bandwidth_gbps=total_bw, bandwidth_available=bw_avail, + bandwidth_requested_gbps=requested_bw, + # Unprivileged IOReport reports every CPU energy channel as 0 mJ on + # some SoCs; propagate the sampler's "ever saw a nonzero delta" flag + # so consumers render n/a instead of a fabricated 0 W. + cpu_power_available=bool(cm.get("cpu_power_available", True)), fans=list(sample.fans), fan_rpms=[f.current for f in sample.fans], fan_available=sample.fan_available, diff --git a/actop/export.py b/actop/export.py index 95b9178..c7bc75d 100644 --- a/actop/export.py +++ b/actop/export.py @@ -49,6 +49,15 @@ # Bandwidth stays decimal GB/s: that is Apple's own unit for the bus (546 # GB/s on M4 Max) and the DCS bucket labels are literally "32GB/s". ("bandwidth_gbps", "memory_bandwidth_gbps"), + # Fabric-side demand aggregate (PMP/AFR BW "AMCC RD+WR"): what the + # memory controllers were asked to move, vs. bandwidth_gbps which is + # what the DRAM pins actually moved. + ("bandwidth_requested_gbps", "memory_bandwidth_requested_gbps"), + # 1 = a real CPU-energy delta was observed at least once this session; + # 0 = the unprivileged-IOReport blind spot on some SoCs, where + # cpu_power_watts reads 0 and must be treated as unknown (powermetrics + # as root is the only true CPU-power source there), not as idle. + ("cpu_power_available", "cpu_power_available"), # Network / disk I/O rates in bytes/s (canonical base unit). ("net_rx_bps", "network_receive_bytes_per_second"), ("net_tx_bps", "network_transmit_bytes_per_second"), diff --git a/actop/models.py b/actop/models.py index e467e6d..0a1bda3 100644 --- a/actop/models.py +++ b/actop/models.py @@ -142,3 +142,13 @@ class SystemSnapshot: ecpu_residency_pct: dict = field(default_factory=_default_residency) pcpu_residency_pct: dict = field(default_factory=_default_residency) gpu_residency_pct: dict = field(default_factory=_default_residency) + # Fabric-side bandwidth demand (PMP/AFR BW "AMCC RD+WR" aggregate), GB/s. + # What the memory controllers were asked to move, vs. bandwidth_gbps + # (above) which is what the DRAM pins actually moved. 0.0 when the + # channel is absent; only meaningful when bandwidth_available is True. + bandwidth_requested_gbps: float = 0.0 + # False while every CPU energy delta has read zero — the unprivileged + # IOReport blind spot on some SoCs (e.g. t8103/M1 Max, where every + # "CPU Energy" channel reports 0 mJ even under full load). Consumers + # MUST render cpu_watts as unavailable, never as a measured 0 W. + cpu_power_available: bool = True diff --git a/actop/sampler.py b/actop/sampler.py index e274027..d484ce0 100644 --- a/actop/sampler.py +++ b/actop/sampler.py @@ -43,11 +43,15 @@ def __init__(self, interval, subsamples=1): ("Energy Model", None), ("CPU Stats", "CPU Core Performance States"), ("GPU Stats", "GPU Performance States"), - # DRAM bandwidth: PMP/DCS BW exposes per-agent residency - # histograms. We only parse the full-range AMCC total; the - # ~90 other channels are subscribed (group-level) but their - # states are skipped in delta() to hold the idle-CPU budget. + # Memory bandwidth: PMP/DCS BW carries the DRAM-controller + # aggregate histogram (channel "RD+WR" on t8103; the real + # pins). PMP/AFR BW carries the per-memory-controller + # aggregate ("AMCC RD+WR") = traffic *requested* through the + # interconnect. Both subscriptions are group-level; delta() + # skips the ~90 other per-agent channels to hold the + # idle-CPU budget. ("PMP", "DCS BW"), + ("PMP", "AFR BW"), ] ) self._interval = interval @@ -59,6 +63,11 @@ def __init__(self, interval, subsamples=1): self._core_counts = _get_core_counts() self._dvfs = get_dvfs_tables_native() self._smc = SMCReader() + # True once any tick showed a nonzero CPU-energy delta. Unprivileged + # IOReport returns 0 mJ for every CPU energy channel on t8103 even + # under full load (verified against powermetrics as root: 32 W real), + # so 0 W must never be presented as a measurement. + self._cpu_power_live = False def sample(self): if self._subsamples <= 1: @@ -283,6 +292,7 @@ def _convert( gpu_freq_mhz = 0 gpu_active_pct = 0 dram_bw_channels = [] + fabric_bw_histogram = None e_cluster_residency_ns = defaultdict(int) p_cluster_residency_ns = defaultdict(int) gpu_state_residencies = [] @@ -334,16 +344,26 @@ def _convert( gpu_state_residencies = list(item.state_residencies) elif item.group == "PMP" and item.subgroup == "DCS BW": - # Total DRAM bandwidth = sum over all AMCC RD+WR instances - # (one per memory-controller die; multi-die SoCs — Ultra — - # expose several). Each instance's histogram is kept as its own - # channel so the per-die averages are *summed*, not pooled into - # one mean (pooling would report a single die's rate as the - # whole-chip total). Per-agent channels (EACC/PACC/AGX/...) are - # skipped in delta() and intentionally not parsed: they hard-cap - # at 32 GB/s and cannot attribute high bandwidth correctly. - if item.channel.startswith("AMCC") and item.channel.endswith("RD+WR"): + # Total DRAM bandwidth = sum over every DRAM-controller + # "RD+WR" aggregate. On t8103 (M1 Max) there is one such + # channel, named exactly "RD+WR"; "AMCC ..." prefixed names + # live under AFR/AFI/AFC BW, not here — filtering for an + # "AMCC" prefix here matched nothing and reported the whole + # domain as unavailable. Per-agent channels (EACC/PACC/AGX) + # are skipped in delta() and intentionally not parsed: they + # hard-cap at 32 GB/s and cannot attribute high bandwidth + # correctly. + if item.channel == "RD+WR" or ( + item.channel.startswith("AMCC") + and item.channel.endswith("RD+WR") + ): dram_bw_channels.append(item.state_residencies) + elif item.group == "PMP" and item.subgroup == "AFR BW": + # Fabric-side aggregate of memory-controller traffic — the + # closest available proxy for bandwidth *demanded* (vs. the + # DCS figure, which is what the DRAM pins actually moved). + if item.channel == "AMCC RD+WR": + fabric_bw_histogram = item.state_residencies # Scale energy to match parsers.py convention: # parsers.py returns cpu_W = energy_mJ / 1000 = energy_J @@ -354,6 +374,8 @@ def _convert( gpu_e = gpu_energy_j * scale ane_e = ane_energy_j * scale package_e = cpu_e + gpu_e + ane_e + if cpu_e > 0.0: + self._cpu_power_live = True cpu_metrics = { "E-Cluster_active": 0, @@ -379,6 +401,14 @@ def _convert( "cpu_W": cpu_e, "gpu_W": gpu_e, "package_W": package_e, + # Honest CPU-power availability: False while every tick's CPU + # energy delta has been zero (unprivileged IOReport on t8103 + # reports 0 mJ for CPU channels — powermetrics as root is the + # only true source). Consumers must render n/a, not 0 W. + "cpu_power_available": self._cpu_power_live, + "bandwidth_requested_gbps": _channel_bandwidth_gbps( + fabric_bw_histogram or [] + ), "e_core": [], "p_core": [], } @@ -429,6 +459,11 @@ def _convert( total_residency = sum(ns for ch in dram_bw_channels for _name, ns in ch) bandwidth_metrics = { "total_gbps": total_gbps, + # Fabric-side demand aggregate (AFR BW "AMCC RD+WR"); may exceed + # total_gbps because request-side counting includes traffic the + # DRAM pins absorbed via cache hits downstream of the controller + # histogram's sampling. + "requested_gbps": _channel_bandwidth_gbps(fabric_bw_histogram or []), # Available only when a channel exists AND carried residency: a # present-but-silent channel would otherwise surface a misleading # 0.0 GB/s instead of hiding the row. @@ -696,12 +731,21 @@ def _resolve_state_freq(name, freq_table): def _keep_states(group, subgroup, channel): """Decide whether delta() should extract per-state residencies for a channel. - The PMP/DCS BW group has ~90 channels of 32 buckets each, but we parse only - the AMCC totals. Skipping state extraction for the rest avoids thousands of - per-cycle ctypes round-trips. All non-PMP groups extract states as before. + The PMP bandwidth groups have ~90+ channels of 32 buckets each, but we + parse only the aggregates: DCS BW's "RD+WR" (DRAM pins) and AFR BW's + "AMCC RD+WR" (fabric-side demand). Skipping state extraction for the + rest avoids thousands of per-cycle ctypes round-trips. All non-PMP + groups extract states as before. """ if group == "PMP": - return channel.startswith("AMCC") + # Admit exactly the channels _convert parses, keyed on subgroup: + # AFC/AFI "AMCC RD+WR" decoys would otherwise burn histogram + # extraction that nothing reads. + if subgroup == "DCS BW": + return channel in ("RD+WR", "AMCC RD+WR") + if subgroup == "AFR BW": + return channel == "AMCC RD+WR" + return False return True diff --git a/actop/tui/app.py b/actop/tui/app.py index 68cbabe..90a0c04 100644 --- a/actop/tui/app.py +++ b/actop/tui/app.py @@ -148,11 +148,15 @@ def _process_display_name(command, max_len=24): GPU GPU util% @freq, die °C ANE Apple Neural Engine util% (estimated) and power RAM Used / total memory (and swap when active) - Mem BW Unified-memory bandwidth in GB/s (hidden if unavailable) + Mem BW Unified-memory bandwidth in GB/s at the DRAM pins; [req X] + marks fabric-side demand above that (cache absorbing the + difference). Hidden if unavailable ↓ In / ↑ Out Network receive / transmit throughput (Network box) ↓ Read / ↑ Write Disk read / write throughput (Disk box) CPU / GPU Live CPU/GPU rail power (W) with an inline sparkline - Package Power Total SoC power draw in watts (CPU + GPU + ANE + other rails) + Package Power Total SoC power draw in watts (CPU + GPU + ANE + other + rails); "(GPU+ANE)" and CPU row "n/a (needs root)" mark + chips whose CPU rail IOReport only opens to root idle/low/mid/high DVFS residency: % of time since the last sample spent idle vs. below 40% / 40-74% / ≥75% of the cluster's max frequency (not just the instantaneous clock). Hidden with diff --git a/actop/tui/widgets.py b/actop/tui/widgets.py index 615d189..b6432c1 100644 --- a/actop/tui/widgets.py +++ b/actop/tui/widgets.py @@ -754,6 +754,10 @@ def __init__(self, config, **kwargs) -> None: # units so the avg/max shown next to "CPU Power 12.3W" or "Mem BW # 120 GB/s" are in watts / GB/s, not percent. self._cpu_w_hist: deque = deque([0] * maxlen, maxlen=maxlen) + # Sticky from the latest snapshot: False while the CPU energy rail + # is unreadable without root (SystemSnapshot.cpu_power_available). + # The CPU power row then renders n/a instead of a fabricated 0.00W. + self._cpu_power_avail = True self._gpu_w_hist: deque = deque([0] * maxlen, maxlen=maxlen) self._pkg_w_hist: deque = deque([0] * maxlen, maxlen=maxlen) self._bw_gbps_hist: deque = deque([0] * maxlen, maxlen=maxlen) @@ -1193,6 +1197,7 @@ def update_metrics(self, message: MetricsUpdated) -> None: self._gpu_hist.append(gpu) self._ane_hist.append(ane_pct) self._ram_hist.append(ram_pct) + self._cpu_power_avail = s.cpu_power_available self._cpu_w_hist.append(s.cpu_watts) self._gpu_w_hist.append(s.gpu_watts) self._sample_count += 1 @@ -1344,7 +1349,10 @@ def update_metrics(self, message: MetricsUpdated) -> None: self._render_power_rows() self._update_stat_row( "#pkgpwr-label", - f"Package Power {s.package_watts:.2f}W", + # "(GPU+ANE)" annotation: on chips whose CPU rail is blind + # unprivileged, this sum is NOT total SoC draw. + f"Package Power {s.package_watts:.2f}W" + + ("" if s.cpu_power_available else " (GPU+ANE)"), self._watt_stats_tiers(self._pkg_w_hist), ) @@ -1357,9 +1365,17 @@ def update_metrics(self, message: MetricsUpdated) -> None: bw_label.display = s.bandwidth_available bw_chart.display = s.bandwidth_available if s.bandwidth_available: + # "[req X]" suffix: fabric-side demand (AFR BW) above what the + # DRAM pins moved means the workload is asking for more bus than + # memory is answering — cache is absorbing the difference. + demand = ( + f" [req {s.bandwidth_requested_gbps:.1f}]" + if s.bandwidth_requested_gbps > s.bandwidth_gbps + 0.5 + else "" + ) self._update_stat_row( "#bw-label", - f"Mem BW {s.bandwidth_gbps:.1f} GB/s", + f"Mem BW {s.bandwidth_gbps:.1f} GB/s{demand}", self._gbps_stats_tiers(self._bw_gbps_hist), ) @@ -1651,7 +1667,7 @@ def _render_power_rows(self) -> None: self._render_power_row( "#cpupwr-row", "CPU", - self._cpu_w_hist[-1], + self._cpu_w_hist[-1] if self._cpu_power_avail else None, self._cpupwr_hist, self._cpu_w_hist, ) @@ -1676,6 +1692,10 @@ def _render_power_row( """ widget = self.query_one(widget_id, Static) avail = widget.size.width + if watts is None: + line = f"{label} n/a (needs root)" + widget.update(line[:avail].ljust(avail) if avail > 0 else line) + return head = f"{label} {watts:.2f}W" tiers = self._watt_stats_tiers(watt_hist) suffix = _fit_suffix( diff --git a/tests/test_sampler.py b/tests/test_sampler.py index faf77c5..c08b7eb 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -1,10 +1,16 @@ import resource import sys import time +from types import SimpleNamespace import pytest -from actop.sampler import SampleResult, create_sampler +from actop.sampler import ( + IOReportSampler, + SampleResult, + _keep_states, + create_sampler, +) pytestmark = pytest.mark.local @@ -166,3 +172,63 @@ def test_sampler_resident_memory_stays_flat_over_many_cycles(): f"resident memory grew {growth_mb:.1f} MB over 2000 sample cycles — " "likely a dropped cf_release in the sample/delta path" ) + + +def test_keep_states_keeps_bandwidth_aggregates_drops_per_agent(): + # The two histograms _convert parses; everything else in PMP is per-agent + # noise that would burn thousands of ctypes round-trips per delta. + assert _keep_states("PMP", "DCS BW", "RD+WR") + assert _keep_states("PMP", "AFR BW", "AMCC RD+WR") + assert not _keep_states("PMP", "DCS BW", "EACC0 RD") + assert not _keep_states("PMP", "DCS BW", "AMCC RD") + assert not _keep_states("PMP", "AFR BW", "AMCC RD") + assert not _keep_states("PMP", "AFC BW", "AMCC RD+WR") + assert _keep_states("GPU Stats", "GPU Performance States", "GPUPH00") + + +def _mk_item(group, subgroup, channel, residencies): + return SimpleNamespace( + group=group, + subgroup=subgroup, + channel=channel, + integer_value=0.0, + unit="", + state_residencies=residencies, + ) + + +def test_convert_matches_dcs_rdwr_and_afr_amcc_rdwr(): + """Regression for the 1.9.13 bandwidth blindness. + + _convert used to filter DCS BW for channels starting with "AMCC" — but + on t8103 the DRAM-pin aggregate is named exactly "RD+WR" and the "AMCC" + totals live under AFR/AFI/AFC BW. The filter matched nothing, so + bandwidth_available was permanently False despite the histograms being + right there. This feeds both aggregates plus per-agent decoys (which + hard-cap at 32 GB/s and must never leak into the total). + """ + s = object.__new__(IOReportSampler) + s._interval = 1.0 + s._dvfs = {"ecpu": [], "pcpu": [], "gpu": []} + s._cpu_power_live = False + s._smc = SimpleNamespace(fan_available=False) + + big = 1_000_000_000 + items = [ + _mk_item("PMP", "DCS BW", "RD+WR", [("32GB/s", big)]), + _mk_item("PMP", "DCS BW", "EACC0 RD", [("32GB/s", 1)]), + _mk_item("PMP", "AFR BW", "AMCC RD+WR", [("32GB/s", big), ("64GB/s", big)]), + _mk_item("PMP", "AFR BW", "AMCC RD", [("32GB/s", big)]), + _mk_item("PMP", "AFC BW", "AMCC RD+WR", [("32GB/s", big)]), + ] + result = s._convert(items, 1.0) + + bw = result.bandwidth_metrics + assert bw["_available"] is True + # Bucket labels are upper edges; the mean is midpoint-weighted + # (0-32 bucket -> 16.0, and 16/48 midpoint pair -> 32.0). + assert bw["total_gbps"] == 16.0 + assert bw["requested_gbps"] == 32.0 + assert result.cpu_metrics["bandwidth_requested_gbps"] == 32.0 + # All-zero CPU energy this tick => the honest flag stays False. + assert result.cpu_metrics["cpu_power_available"] is False From 057fdfddaee7b4ddce8fab17319f7d42d06bbbca Mon Sep 17 00:00:00 2001 From: Catalin Date: Sat, 10 Oct 2026 20:32:19 +0300 Subject: [PATCH 3/5] chore(release): version 1.9.13 with dvfs/bandwidth/cpu-honesty notes --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ README.md | 9 +++++++-- pyproject.toml | 2 +- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e39c322..fec38b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,44 @@ This project follows a Keep a Changelog-style format and uses version tags for r ## [Unreleased] +## [1.9.13] - 2026-10-10 + +### Fixed + +- **GPU DVFS tables on Apple M1 Max are no longer CPU clocks wearing a GPU + label.** `_classify_dvfs_tables` picked buckets by table length alone, and + the M1 Max pmgr domain hands it a 15-entry *P-core* SRAM table + (`voltage-states5-sram`, ceiling 3228 MHz) as "gpu" while the real 7-entry + GPU table (`voltage-states9`, 0-1296 MHz) fell outside the 10-20-entry + window. Every GPU clock, DVFS state name (`P5` resolved to 2676 MHz), + residency histogram, and derived TFLOPS figure was CPU telemetry. M1 Max + now maps its buckets through a hardware-validated override + (`voltage-states1-sram` → E, `voltage-states5-sram` → P, `voltage-states9` + → GPU); the shape heuristic additionally refuses any GPU candidate whose + max is not strictly below the P-core max, so other SoCs can no longer + mislabel either — an unrecognized chip degrades to the IOAccelerator + utilization fallback instead of publishing false clocks. +- **Memory bandwidth is reported instead of permanently "unavailable".** + The sampler subscribed to `(PMP, "DCS BW", channel startswith "AMCC")`, + but on t8103 the DRAM-pin aggregate is named exactly `RD+WR` (the `AMCC` + totals live under AFR/AFI/AFC BW). The filter matched nothing, so + `bandwidth_available` was False forever and the Mem BW row, chart, alerts, + and JSON field were dead. `RD+WR` is now the headline (GB/s at the DRAM + pins, 400 GB/s class on M1 Max) and the AFR `AMCC RD+WR` fabric aggregate + is exposed as `bandwidth_requested_gbps` — when demand exceeds pinned + traffic the TUI suffixes `[req X]`, showing that L2 is absorbing the gap + (sushi-style LLM decode: ~81-128 GB/s DRAM vs ~107 GB/s fabric, ~30%). + +- **CPU power is honest where IOReport keeps it root-only.** Unprivileged + IOReport reports every `CPU Energy` channel as exactly 0 mJ on this SoC + even under 8-core full load (cross-checked against root `powermetrics`: + 32.4 W where actop showed 0.00 W — a fabricated zero indistinguishable + from idle). The sampler now records whether any tick ever saw a nonzero + CPU energy delta and exports it as `cpu_power_available`; the TUI renders + the CPU row as `n/a (needs root)` and annotates the package total as + `(GPU+ANE)` while the rail is blind. Run under root (or the planned sudo + mode) and the real CPU watts appear with no other change. + ## [1.9.12] - 2026-08-23 ### Fixed diff --git a/README.md b/README.md index 108e875..b8282d3 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ No sudo required. Degrades to `Unknown` if the ObjC runtime call fails. | Signal | Source | Notes | | --- | --- | --- | -| CPU/GPU/ANE power (W) | IOReport Energy Model | nJ per sample interval → watts | +| CPU/GPU/ANE power (W) | IOReport Energy Model | nJ per sample interval → watts. On SoCs where the CPU energy rail is root-gated (e.g. M1 Max), unprivileged reads return 0 mJ; the CPU row then renders `n/a (needs root)` and the package total is annotated `(GPU+ANE)` — never a fabricated zero (`cpu_power_available`) | | Per-core frequency (MHz) | IOReport residency + DVFS tables | Weighted average of active P-states | | Per-core activity (%) | IOReport CPU Core Performance States | Via `CoreSample` (residency-weighted active%) | | GPU frequency and activity | IOReport GPU Performance States | Weighted average of GPUPH residencies | @@ -275,7 +275,7 @@ No sudo required. Degrades to `Unknown` if the ObjC runtime call fails. | RAM / swap | Native `ctypes` (`host_statistics64` + `XSWUsage`) | `total - available` for used. API/export carry exact **bytes**; the TUI displays **GiB** (2³⁰). Bandwidth stays decimal `GB/s` — Apple's own unit for the bus | | SoC profile | `sysctl` brand → 16 M1–M4 profiles | Tier fallbacks for unknown chips | | Top processes | Native `ctypes` (`proc_listpids`/`proc_pidinfo`) | Optional `--proc-filter` regex | -| Bandwidth | IOReport (when available) | N/A if DCS counters not exposed | +| Bandwidth | IOReport PMP `DCS BW` / `RD+WR` (DRAM pins) + PMP `AFR BW` / `AMCC RD+WR` (fabric demand) | N/A if DCS counters not exposed; TUI suffixes `[req X]` when fabric demand exceeds pinned traffic (cache absorbing the gap) | | Thermal pressure | `NSProcessInfo.thermalState` via ObjC runtime | Nominal / Fair / Serious / Critical | ## Architecture @@ -349,6 +349,11 @@ graph TD ## Troubleshooting - **Bandwidth shows N/A**: IOReport does not expose memory bandwidth counters on all SoCs. +- **CPU power reads `n/a (needs root)`**: on some SoCs (M1 Max among them) + unprivileged IOReport exposes the CPU energy channels but reports 0 mJ for + them even at full load. GPU/ANE rails and package-relative charts are + unaffected; run `sudo actop` to light the CPU row up. actop deliberately + shows `n/a` rather than a plausible-looking 0 W. - **Thermal shows "Unknown"**: ObjC runtime failed to read `NSProcessInfo.thermalState` (unexpected on macOS 12+). - **Frequencies show 0 MHz**: DVFS table discovery failed. File an issue with `sysctl -n machdep.cpu.brand_string` output. - **Metric differences vs other tools**: expected due to sampling window and source timing differences. diff --git a/pyproject.toml b/pyproject.toml index 87cf830..bfce60b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "actop" -version = "1.9.12" +version = "1.9.13" description = "Apple Silicon (M1–M4) power, GPU, ANE & memory-bandwidth monitor — sudoless TUI + Python API for profiling local LLM / MLX / CoreML inference" readme = "README.md" license = { text = "MIT" } From 337f4b7a6c3eb82f9420836440d78281496c60d8 Mon Sep 17 00:00:00 2001 From: Catalin Date: Sat, 10 Oct 2026 21:14:02 +0300 Subject: [PATCH 4/5] =?UTF-8?q?fix(tui):=20reword=20CPU=20rail=20cause=20?= =?UTF-8?q?=E2=80=94=20root=20is=20blind=20too?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Concurrent-probe experiment (root powermetrics streaming while a root IOReport pull-subscriber sampled the same window): the CPU energy channels advance only while a privileged powermetrics stream arms them. Every solo root actop run still reads 0 mJ, so 'n/a (needs root)' was itself a fabricated premise. The row now reads 'n/a (no data)', the legend, README signal-source row, troubleshooting bullet, sampler and models docstrings, and the CHANGELOG entry state the verified behavior: powermetrics-only, sudo does not light the rail. --- CHANGELOG.md | 13 +++++++------ README.md | 13 +++++++------ actop/models.py | 9 +++++---- actop/sampler.py | 15 ++++++++------- actop/tui/app.py | 5 +++-- actop/tui/widgets.py | 6 ++++-- 6 files changed, 34 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fec38b4..c5f07e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,15 +34,16 @@ This project follows a Keep a Changelog-style format and uses version tags for r traffic the TUI suffixes `[req X]`, showing that L2 is absorbing the gap (sushi-style LLM decode: ~81-128 GB/s DRAM vs ~107 GB/s fabric, ~30%). -- **CPU power is honest where IOReport keeps it root-only.** Unprivileged +- **CPU power is honest where IOReport never delivers it.** Unprivileged IOReport reports every `CPU Energy` channel as exactly 0 mJ on this SoC even under 8-core full load (cross-checked against root `powermetrics`: 32.4 W where actop showed 0.00 W — a fabricated zero indistinguishable - from idle). The sampler now records whether any tick ever saw a nonzero - CPU energy delta and exports it as `cpu_power_available`; the TUI renders - the CPU row as `n/a (needs root)` and annotates the package total as - `(GPU+ANE)` while the rail is blind. Run under root (or the planned sudo - mode) and the real CPU watts appear with no other change. + from idle). Root does not help: concurrent-probe experiments show the + counters only advance while a privileged powermetrics stream arms them, + and every solo root run reads 0 mJ. The sampler now records whether any + tick ever saw a nonzero CPU energy delta and exports it as + `cpu_power_available`; the TUI renders the CPU row as `n/a (no data)` + and annotates the package total `(GPU+ANE)` while the rail is blind. ## [1.9.12] - 2026-08-23 diff --git a/README.md b/README.md index b8282d3..f603dad 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ No sudo required. Degrades to `Unknown` if the ObjC runtime call fails. | Signal | Source | Notes | | --- | --- | --- | -| CPU/GPU/ANE power (W) | IOReport Energy Model | nJ per sample interval → watts. On SoCs where the CPU energy rail is root-gated (e.g. M1 Max), unprivileged reads return 0 mJ; the CPU row then renders `n/a (needs root)` and the package total is annotated `(GPU+ANE)` — never a fabricated zero (`cpu_power_available`) | +| CPU/GPU/ANE power (W) | IOReport Energy Model | nJ per sample interval → watts. On some SoCs (M1 Max) the CPU energy rail is never delivered to *any* IOReport client — it reads 0 mJ even as root; the CPU row then renders `n/a (no data)` and the package total is annotated `(GPU+ANE)` — never a fabricated zero (`cpu_power_available`) | | Per-core frequency (MHz) | IOReport residency + DVFS tables | Weighted average of active P-states | | Per-core activity (%) | IOReport CPU Core Performance States | Via `CoreSample` (residency-weighted active%) | | GPU frequency and activity | IOReport GPU Performance States | Weighted average of GPUPH residencies | @@ -349,11 +349,12 @@ graph TD ## Troubleshooting - **Bandwidth shows N/A**: IOReport does not expose memory bandwidth counters on all SoCs. -- **CPU power reads `n/a (needs root)`**: on some SoCs (M1 Max among them) - unprivileged IOReport exposes the CPU energy channels but reports 0 mJ for - them even at full load. GPU/ANE rails and package-relative charts are - unaffected; run `sudo actop` to light the CPU row up. actop deliberately - shows `n/a` rather than a plausible-looking 0 W. +- **CPU power reads `n/a (no data)`**: on some SoCs (M1 Max among them) the + CPU energy channels read 0 mJ for every IOReport client — including root + under full load; the counters only move while a privileged `powermetrics` + stream is running, and actop does not shell out to it. GPU/ANE rails and + package-relative charts are unaffected. actop deliberately shows `n/a` + rather than a plausible-looking 0 W. - **Thermal shows "Unknown"**: ObjC runtime failed to read `NSProcessInfo.thermalState` (unexpected on macOS 12+). - **Frequencies show 0 MHz**: DVFS table discovery failed. File an issue with `sysctl -n machdep.cpu.brand_string` output. - **Metric differences vs other tools**: expected due to sampling window and source timing differences. diff --git a/actop/models.py b/actop/models.py index 0a1bda3..00e2203 100644 --- a/actop/models.py +++ b/actop/models.py @@ -147,8 +147,9 @@ class SystemSnapshot: # (above) which is what the DRAM pins actually moved. 0.0 when the # channel is absent; only meaningful when bandwidth_available is True. bandwidth_requested_gbps: float = 0.0 - # False while every CPU energy delta has read zero — the unprivileged - # IOReport blind spot on some SoCs (e.g. t8103/M1 Max, where every - # "CPU Energy" channel reports 0 mJ even under full load). Consumers - # MUST render cpu_watts as unavailable, never as a measured 0 W. + # False while every CPU energy delta has read zero. On some SoCs (e.g. + # t8103/M1 Max) the "CPU Energy" channels are never delivered to any + # IOReport client — 0 mJ even as root under full load; only privileged + # powermetrics sees the rail. Consumers MUST render cpu_watts as + # unavailable, never as a measured 0 W. cpu_power_available: bool = True diff --git a/actop/sampler.py b/actop/sampler.py index d484ce0..0d1ed6b 100644 --- a/actop/sampler.py +++ b/actop/sampler.py @@ -63,10 +63,11 @@ def __init__(self, interval, subsamples=1): self._core_counts = _get_core_counts() self._dvfs = get_dvfs_tables_native() self._smc = SMCReader() - # True once any tick showed a nonzero CPU-energy delta. Unprivileged - # IOReport returns 0 mJ for every CPU energy channel on t8103 even - # under full load (verified against powermetrics as root: 32 W real), - # so 0 W must never be presented as a measurement. + # True once any tick showed a nonzero CPU-energy delta. On t8103 no + # IOReport client ever sees one — root reads 0 mJ too, and the + # counters only move while a privileged powermetrics stream is + # arming them (verified 2026-10-10 with concurrent probes), so 0 W + # must never be presented as a measurement. self._cpu_power_live = False def sample(self): @@ -402,9 +403,9 @@ def _convert( "gpu_W": gpu_e, "package_W": package_e, # Honest CPU-power availability: False while every tick's CPU - # energy delta has been zero (unprivileged IOReport on t8103 - # reports 0 mJ for CPU channels — powermetrics as root is the - # only true source). Consumers must render n/a, not 0 W. + # energy delta has been zero. On t8103 the CPU rail is silent to + # every IOReport client including root — privileged powermetrics + # is the only true source. Consumers must render n/a, not 0 W. "cpu_power_available": self._cpu_power_live, "bandwidth_requested_gbps": _channel_bandwidth_gbps( fabric_bw_histogram or [] diff --git a/actop/tui/app.py b/actop/tui/app.py index 90a0c04..047bcb1 100644 --- a/actop/tui/app.py +++ b/actop/tui/app.py @@ -155,8 +155,9 @@ def _process_display_name(command, max_len=24): ↓ Read / ↑ Write Disk read / write throughput (Disk box) CPU / GPU Live CPU/GPU rail power (W) with an inline sparkline Package Power Total SoC power draw in watts (CPU + GPU + ANE + other - rails); "(GPU+ANE)" and CPU row "n/a (needs root)" mark - chips whose CPU rail IOReport only opens to root + rails); "(GPU+ANE)" and CPU row "n/a (no data)" mark + chips whose CPU rail IOReport never delivers (even root + reads 0; only powermetrics sees it) idle/low/mid/high DVFS residency: % of time since the last sample spent idle vs. below 40% / 40-74% / ≥75% of the cluster's max frequency (not just the instantaneous clock). Hidden with diff --git a/actop/tui/widgets.py b/actop/tui/widgets.py index b6432c1..1ce70e1 100644 --- a/actop/tui/widgets.py +++ b/actop/tui/widgets.py @@ -755,7 +755,9 @@ def __init__(self, config, **kwargs) -> None: # 120 GB/s" are in watts / GB/s, not percent. self._cpu_w_hist: deque = deque([0] * maxlen, maxlen=maxlen) # Sticky from the latest snapshot: False while the CPU energy rail - # is unreadable without root (SystemSnapshot.cpu_power_available). + # is unreadable by any IOReport subscriber on some chips (t8103: + # root reads 0 too; only powermetrics sees the rail — verified + # 2026-10-10 via concurrent-probe experiment). # The CPU power row then renders n/a instead of a fabricated 0.00W. self._cpu_power_avail = True self._gpu_w_hist: deque = deque([0] * maxlen, maxlen=maxlen) @@ -1693,7 +1695,7 @@ def _render_power_row( widget = self.query_one(widget_id, Static) avail = widget.size.width if watts is None: - line = f"{label} n/a (needs root)" + line = f"{label} n/a (no data)" widget.update(line[:avail].ljust(avail) if avail > 0 else line) return head = f"{label} {watts:.2f}W" From fd013c9c934b1950daf06d66cdab995863205025 Mon Sep 17 00:00:00 2001 From: Catalin Date: Sat, 10 Oct 2026 21:32:20 +0300 Subject: [PATCH 5/5] test(tui): pin rail-honesty and demand rendering cpu_power_available=False must render 'CPU n/a (no data)' (never the fabricated 0.00W) with the '(GPU+ANE)' package annotation, the row must return to watts when the rail is live, and #bw-label must suffix '[req X]' only when fabric demand exceeds pinned traffic by >0.5 GB/s. --- tests/test_dashboard_metrics.py | 69 +++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/test_dashboard_metrics.py b/tests/test_dashboard_metrics.py index c978864..0b627e3 100644 --- a/tests/test_dashboard_metrics.py +++ b/tests/test_dashboard_metrics.py @@ -93,6 +93,8 @@ def _snapshot( disk_read_bps: float = 0.0, disk_write_bps: float = 0.0, disk_available: bool = False, + bandwidth_requested_gbps: float = 0.0, + cpu_power_available: bool = True, ) -> SystemSnapshot: fans = [] if fans is None else fans idle_residency = {"idle": 100, "low": 0, "mid": 0, "high": 0} @@ -141,6 +143,8 @@ def _snapshot( disk_read_bps=disk_read_bps, disk_write_bps=disk_write_bps, disk_available=disk_available, + bandwidth_requested_gbps=bandwidth_requested_gbps, + cpu_power_available=cpu_power_available, ) @@ -880,3 +884,68 @@ async def _run(): scroll_y, max_scroll_y = asyncio.run(_run()) assert max_scroll_y > 0 assert scroll_y == 5 + + +# --- Rail-honesty and fabric-demand rendering (t8103 follow-up) ---------- + +def test_cpu_power_row_reads_no_data_when_rail_blind(): + # On t8103 every "CPU Energy" channel is 0 mJ for every IOReport client + # (root included), so rendering cpu_watts as "CPU 0.00W" is a fabricated + # measurement indistinguishable from idle. When the snapshot flags the + # rail blind, the row must say so, and the package headline — which then + # sums only GPU+ANE+misc — must carry the annotation. + async def _run(): + dash = HardwareDashboard(config=_config()) + app = _Host(dash) + async with app.run_test(size=(160, 50)) as pilot: + dash.update_metrics( + MetricsUpdated(_snapshot(10.0, True, cpu_power_available=False)) + ) + await pilot.pause() + return ( + str(dash.query_one("#cpupwr-row", Static).render()), + str(dash.query_one("#pkgpwr-label", Static).render()), + ) + + cpu_row, pkg = asyncio.run(_run()) + assert "CPU n/a (no data)" in cpu_row + assert "0.00W" not in cpu_row + assert "(GPU+ANE)" in pkg + + +def test_cpu_power_row_restores_when_rail_live(): + # The blindness flag is the sampler's; once any tick saw a nonzero delta, + # snapshots arrive available=True and the watt headline returns. + async def _run(): + dash = HardwareDashboard(config=_config()) + app = _Host(dash) + async with app.run_test(size=(160, 50)) as pilot: + dash.update_metrics( + MetricsUpdated(_snapshot(10.0, True, cpu_power_available=False)) + ) + await pilot.pause() + dash.update_metrics( + MetricsUpdated(_snapshot(10.0, True, cpu_power_available=True)) + ) + await pilot.pause() + return str(dash.query_one("#cpupwr-row", Static).render()) + + assert "CPU 8.00W" in asyncio.run(_run()) + + +def test_bw_label_suffixes_fabric_demand_above_the_pins(): + # LLM decode drives the fabric above DRAM-pin traffic (cache absorbs the + # difference); that demand must be surfaced as "[req X]", and must vanish + # when the two agree within the 0.5 GB/s margin. + async def _run(req): + dash = HardwareDashboard(config=_config()) + app = _Host(dash) + async with app.run_test(size=(160, 50)) as pilot: + dash.update_metrics( + MetricsUpdated(_snapshot(80.0, True, bandwidth_requested_gbps=req)) + ) + await pilot.pause() + return str(dash.query_one("#bw-label", Static).render()) + + assert "[req 107.0]" in asyncio.run(_run(107.0)) + assert "[req" not in asyncio.run(_run(80.3))