From f5b8909de216d8e0766984a3948aa8663096c6a9 Mon Sep 17 00:00:00 2001 From: Tom Elliott Date: Tue, 27 Oct 2026 09:02:12 -0500 Subject: [PATCH] Add the Performance Test task, mode=perftest (fos#55) A slow deploy is limited by one of three stages: the network, the CPU (decompression) or the target disk. Users had to be walked through debug mode to find which. fog.perftest times each stage and names the slowest. - Disk: a direct (uncached) sequential read of the target disk. Read only; nothing is written. - Network: reads the largest stored image over NFS, mounted read-only. The server sends a non-imaging task the dev/ capture share, so the stored images are its parent. - CPU: decompresses a sample of that image with the deploy's own decompressor (zstdmt -dc or pigz -dc). - Rates are uncompressed MB/s and GB/min, the unit partclone shows; the network rate is scaled by the sample's compression ratio. - The result is posted to service/perftest.php before the task is completed, so the server can store it on the host. A failed post is shown, not fatal. The logic is in perftest-funcs.sh so tests/checks/perftest.sh can run it off-target (35 cases; verified red by removing the dev/ mapping and by widening the sample glob). Co-Authored-By: Claude Opus 5.5 --- .../board/FOG/FOS/rootfs_overlay/bin/fog | 3 + .../FOG/FOS/rootfs_overlay/bin/fog.perftest | 161 ++++++++++++++++++ .../usr/share/fog/lib/perftest-funcs.sh | 143 ++++++++++++++++ CLAUDE.md | 1 + tests/README.md | 8 + tests/checks/perftest.sh | 118 +++++++++++++ 6 files changed, 434 insertions(+) create mode 100755 Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.perftest create mode 100644 Buildroot/board/FOG/FOS/rootfs_overlay/usr/share/fog/lib/perftest-funcs.sh create mode 100644 tests/checks/perftest.sh diff --git a/Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog b/Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog index e03684d..de67b34 100755 --- a/Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog +++ b/Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog @@ -80,6 +80,9 @@ if [[ -n $mode && $mode != +(*debug*) && -z $type ]]; then enrollsbforce) fog.enrollsbforce ;; + perftest) + fog.perftest + ;; *) handleError "Fatal Error: Unknown mode :: $mode ($0)\n Args Passed: $*" ;; diff --git a/Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.perftest b/Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.perftest new file mode 100755 index 0000000..945b3fa --- /dev/null +++ b/Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.perftest @@ -0,0 +1,161 @@ +#!/bin/bash +# +# Performance Test task (fos#55): times the three stages a deploy runs +# through -- disk, network, CPU -- names the slowest, shows the result and +# reports it to the server so it stays on the host's record. Nothing is +# written to the disk and nothing is written to the storage node: the disk +# test only reads, and /images is mounted read-only. +. /usr/share/fog/lib/funcs.sh +. /usr/share/fog/lib/perftest-funcs.sh +clearScreen +displayBanner + +# How much each test moves. Large enough to get past caches and ramp-up, small +# enough that the whole task takes a minute or two on slow hardware. The +# decompression sample is held in RAM (/tmp is tmpfs), so it stays modest. +DISK_MIB=2048 +NET_MIB=1024 +CPU_MIB=256 + +dots "Locating disks" +disks="" +disk="" +hd="" +getHardDisk +if [[ -z $hd ]]; then + echo "Failed" + debugPause + handleError "No hard disks found! ($0)\n Args Passed: $*" +fi +echo "Done" +debugPause + +dots "Attempting to checkin" +res="" +count=0 +sysuuid=$(dmidecode -s system-uuid) +sysuuid=${sysuuid,,} +while [[ $res != "##@GO" ]]; do + callServer "${web}service/Pre_Stage1.php" "sysuuid=${sysuuid}&mac=$mac" + res="$serverBody" + if [[ $count -ge 10 ]]; then + echo "Failed" + debugPause + handleError "Could not checkin: ${serverReason:-last reply was \"$res\"} ($0)\n Args Passed: $*" + fi + let count+=1 + usleep 5000000 +done +echo "Done" +debugPause + +# Disk: a direct (uncached) sequential read of the target disk. +dots "Testing disk read ($hd)" +diskmbps=0 +if perfReadTest "$hd" "$DISK_MIB" 1; then + diskmbps=$(perfRate "$PERF_BYTES" "$PERF_NS") + echo "Done" +else + echo "Failed" +fi +debugPause + +# Network and CPU both need a captured image to read. fog.mount is not used: +# it mounts only for type=up/down, and this mount must be read-only. +netmbps=0 +cpumbps=0 +ratio100=100 +fmt="" +sample="" +[[ ! -d /images ]] && mkdir -p /images >/dev/null 2>&1 +umount /images >/dev/null 2>&1 +# The server sends a non-imaging task the capture share, /dev/ +# (BootMenuBase::getTasking). The stored images a deploy reads are its parent. +imgroot=$(perfImageRoot "$storage") +dots "Mounting storage read-only" +# -t nfs is load-bearing, see the comment in fog.mount. +if mount -t nfs -o ro,nolock,proto=tcp,rsize=32768,intr,noatime "$imgroot" /images >/tmp/mount-output 2>&1; then + echo "Done" + sample=$(perfPickSample /images) +else + echo "Failed" + echo " $(tr -d \\0 /dev/null 2>&1 + if perfDecompressTest /tmp/perf.sample "$fmt"; then + cpumbps=$(perfRate "$PERF_OUT" "$PERF_NS") + ratio100=$(perfRatio100 "$PERF_IN" "$PERF_OUT") + echo "Done" + else + echo "Failed" + fi + rm -f /tmp/perf.sample + debugPause + fi + # The network moves compressed bytes; scale to what they unpack to. + netmbps=$(perfScale "$netcompressed" "$ratio100") +fi +umount /images >/dev/null 2>&1 + +bottleneck=$(perfBottleneck "$diskmbps" "$netmbps" "$cpumbps") + +perfLine() { + local label="$1" mbps="$2" note="$3" + if [[ $mbps -gt 0 ]]; then + printf " %-12s %6s MB/s %5s GB/min%s\n" "$label" "$mbps" "$(perfGBmin "$mbps")" "$note" + else + printf " %-12s %s\n" "$label" "not measured" + fi +} +echo +echo " +-----------------------------------------+" +echo " | Performance Test (uncompressed rates) |" +echo " +-----------------------------------------+" +perfLine "Disk read" "$diskmbps" " ($hd)" +perfLine "Network" "$netmbps" +perfLine "Decompress" "$cpumbps" "${fmt:+ ($fmt)}" +[[ -z $sample ]] && echo " No captured image on the storage node, so network and CPU were skipped." +if [[ $ratio100 -ne 100 ]]; then + echo " Compression ratio of the sample: $(( ratio100 / 100 )).$(( ratio100 % 100 / 10 )):1" +fi +if [[ -n $bottleneck ]]; then + echo + echo " A deploy to this machine is limited by: $bottleneck" +fi +echo + +# Reported BEFORE the task is completed, so the server can still tie the +# result to this task. A failed report is a warning, not an abort: the +# measurement is on screen either way. +dots "Sending results to the FOG server" +sample64=$(echo -n "${sample#/images/}" | base64 | tr -d '\n') +kernel=$(uname -r) +initver=$(awk -F= '/^export initversion=/{print $2}' /usr/share/fog/lib/funcs.sh) +callServer "${web}service/perftest.php" "mac=${mac}&disk=${hd}&diskmbps=${diskmbps}&netmbps=${netmbps}&cpumbps=${cpumbps}&ratio=${ratio100}&format=${fmt}&sample=${sample64}&bottleneck=${bottleneck}&kernel=${kernel}&initversion=${initver}" +if [[ $serverBody == "##" ]]; then + echo "Done" +else + echo "Failed" + echo " ${serverReason:-$serverBody}" +fi +debugPause +echo " * Press [Enter] key to continue" +read -t 300 -p "" +. /bin/fog.nonimgcomplete diff --git a/Buildroot/board/FOG/FOS/rootfs_overlay/usr/share/fog/lib/perftest-funcs.sh b/Buildroot/board/FOG/FOS/rootfs_overlay/usr/share/fog/lib/perftest-funcs.sh new file mode 100644 index 0000000..a70377e --- /dev/null +++ b/Buildroot/board/FOG/FOS/rootfs_overlay/usr/share/fog/lib/perftest-funcs.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# +# Imaging performance test (fos#55). A deploy moves an image through three +# stages -- the network (NFS read from the storage node), the CPU (zstd or +# pigz decompression) and the target disk -- and runs no faster than the +# slowest one. These functions time each stage on its own so the slowest can +# be named. bin/fog.perftest runs them; tests/checks/perftest.sh exercises +# them off-target. +# +# Every rate is reported in UNCOMPRESSED megabytes per second, the unit +# partclone's own GB/min display uses. The network and CPU stages move +# compressed bytes, so they are scaled by the sample's compression ratio; +# without that, a 2:1 image would make the network look half as fast as it +# is next to the disk. + +# Megabytes (10^6 bytes) per second from a byte count and a duration in +# nanoseconds. Integer only: bash has no floating point, and a whole MB/s is +# all the comparison needs. A zero duration yields 0 rather than dividing by +# zero. +perfRate() { + local bytes="$1" ns="$2" + [[ -z $bytes || -z $ns || $ns -le 0 ]] && { echo 0; return; } + echo $(( bytes * 1000 / ns )) +} + +# GB/min from MB/s, to one decimal place, for the line people compare with +# partclone's display. +perfGBmin() { + local mbps="$1" + local tenths=$(( mbps * 60 / 100 )) + echo "$(( tenths / 10 )).$(( tenths % 10 ))" +} + +# The NFS source of the stored images, from the storage= a non-imaging task +# receives. The server sends such a task ://dev/, the capture +# share; the stored images are its parent. Anything not ending in dev/ is +# returned as it is. +perfImageRoot() { + local s="${1%/}" + [[ $s == */dev ]] && s="${s%/dev}" + echo "$s/" +} + +# The largest captured image file under the mounted storage root, or nothing. +# Captured partitions are dp.img (pigz, one file) or +# dp.img.NNN (zstd, split into 200M pieces). Captures still in +# progress sit one level deeper, at dev//, which the glob's depth never +# reaches: a half-written file says nothing about the images a deploy reads. +perfPickSample() { + local root="$1" f size best="" bestsize=0 + # A glob, not find: the init carries busybox find, which has no -printf. + for f in "$root"/*/d*p*.img "$root"/*/d*p*.img.[0-9][0-9][0-9]; do + [[ -f $f ]] || continue + size=$(stat -c %s "$f" 2>/dev/null) + [[ -n $size && $size -gt $bestsize ]] && { best=$f; bestsize=$size; } + done + echo "$best" +} + +# zstd, gzip or unknown, from the first four bytes. pigz writes ordinary gzip. +perfFormat() { + local magic + magic=$(od -An -tx1 -N4 "$1" 2>/dev/null | tr -d ' \n') + case $magic in + 28b52ffd) echo zstd ;; + 1f8b*) echo gzip ;; + *) echo unknown ;; + esac +} + +# The decompressor a deploy uses for a given format, matching writeImage(): +# zstdmt -dc for zstd images, pigz -dc for gzip ones. +perfDecompressor() { + case $1 in + zstd) echo "zstdmt -dc" ;; + gzip) echo "pigz -dc" ;; + esac +} + +# Reads up to MiB from into /dev/null and sets PERF_BYTES and +# PERF_NS. = 1 adds iflag=direct, which bypasses the page cache so a +# disk read measures the disk and not RAM. Returns dd's status. +perfReadTest() { + local src="$1" mib="$2" direct="$3" flag="" start end out rc + [[ $direct == 1 ]] && flag="iflag=direct" + start=$(date +%s%N) + out=$(dd if="$src" of=/dev/null bs=1M count="$mib" $flag 2>&1) + rc=$? + end=$(date +%s%N) + PERF_NS=$(( end - start )) + # dd's summary line starts with the byte count it actually moved, which is + # less than MiB when the source is shorter. + PERF_BYTES=$(echo "$out" | awk '/bytes/ {print $1; exit}') + [[ -z $PERF_BYTES ]] && PERF_BYTES=0 + return $rc +} + +# Decompresses (a compressed fragment already in RAM) with the +# deploy's own decompressor and sets PERF_IN, PERF_OUT and PERF_NS. The +# fragment ends mid-stream, so the decompressor's exit status is expected to +# be non-zero and is ignored; what it produced before the cut is what is +# measured. +perfDecompressTest() { + local sample="$1" fmt="$2" cmd start end + cmd=$(perfDecompressor "$fmt") + PERF_IN=$(stat -c %s "$sample" 2>/dev/null) + [[ -z $cmd || -z $PERF_IN ]] && { PERF_OUT=0; PERF_NS=0; return 1; } + start=$(date +%s%N) + PERF_OUT=$($cmd < "$sample" 2>/dev/null | wc -c) + end=$(date +%s%N) + PERF_NS=$(( end - start )) + return 0 +} + +# Uncompressed output bytes per compressed input byte, times 100, so a 2.5:1 +# image gives 250. 100 (no scaling) when either count is missing. +perfRatio100() { + local in="$1" out="$2" + [[ -z $in || -z $out || $in -le 0 || $out -le 0 ]] && { echo 100; return; } + echo $(( out * 100 / in )) +} + +# A compressed-byte rate scaled to the uncompressed bytes it unpacks to. +perfScale() { + echo $(( $1 * $2 / 100 )) +} + +# Names the slowest of the stages that were measured. Arguments are +# uncompressed MB/s for disk, network and cpu; an empty or 0 value means the +# stage was not measured and is left out. +perfBottleneck() { + local -a names=(disk network cpu) + local -a rates=("$1" "$2" "$3") + local i best="" bestrate="" + for i in 0 1 2; do + [[ -z ${rates[$i]} || ${rates[$i]} -le 0 ]] && continue + if [[ -z $bestrate || ${rates[$i]} -lt $bestrate ]]; then + best=${names[$i]} + bestrate=${rates[$i]} + fi + done + echo "$best" +} diff --git a/CLAUDE.md b/CLAUDE.md index ae3eff4..ad79e9f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -179,6 +179,7 @@ tests/checks/mdraid-gate.sh # md arrays auto-assemble only with mdraid= tests/checks/multicast-receiver.sh # udp-receiver gets --start-timeout from mcastMaxWait; an empty stream says no sender answered (fos#48) tests/checks/fixparts-binary-present.sh # fs.patch builds fixparts, which runFixparts() calls (fos#194) +tests/checks/perftest.sh # Performance Test task: rates, sampled image, dev/ share, bottleneck (fos#55) tests/checks/package-mirrors.sh # build.sh's package mirror fallback and hash enforcement ``` diff --git a/tests/README.md b/tests/README.md index 29d00de..dd4ec3e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -308,6 +308,14 @@ tests/checks/fixparts-binary-present.sh # it. A symbol with no Kconfig definition is # dropped silently, which is how every init from # 2023-12 on shipped without it. See fos#194 +tests/checks/perftest.sh + # fog.perftest (Performance Test task): MB/s and + # GB/min arithmetic, the dev/ share mapped to the + # stored images, the sampled image (never a + # capture in progress), real zstdmt/pigz on a + # stream cut mid-way, scaling compressed rates + # to uncompressed, and which stage it blames. + # See fos#55 tests/checks/package-mirrors.sh # build.sh's package-mirror seeding: resolves each # package's version/source/site out of its own .mk diff --git a/tests/checks/perftest.sh b/tests/checks/perftest.sh new file mode 100644 index 0000000..a34a388 --- /dev/null +++ b/tests/checks/perftest.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# +# Assertion harness for the Performance Test task (fos#55). +# +# tests/checks/perftest.sh +# +# fog.perftest times the disk, the network and the CPU separately and names +# the slowest. What can go wrong without real hardware is the arithmetic and +# the choices around it: the unit conversion, the image it samples, the share +# it mounts, the decompressor it times, and which stage it blames. Those live +# in perftest-funcs.sh, which this sources as shipped. +# +# The decompression cases use the host's real zstdmt and pigz on a really +# truncated stream, because a deploy's sample is cut mid-stream and the +# decompressor's behavior there is the thing being relied on. + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OVERLAY="$HERE/../../Buildroot/board/FOG/FOS/rootfs_overlay" +LIB="$OVERLAY/usr/share/fog/lib/perftest-funcs.sh" + +fails=0 +checked=0 + +ck() { + checked=$((checked + 1)) + if [[ $2 != 1 ]]; then + echo "FAIL $1" + fails=$((fails + 1)) + fi +} +eq() { [[ $1 == "$2" ]] && echo 1 || echo 0; } + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +. "$LIB" + +# 1. Rates: 10^9 bytes in one second is 1000 MB/s; no duration is 0, not a +# division error. +ck "1a perfRate 1e9 B / 1 s = 1000" "$(eq "$(perfRate 1000000000 1000000000)" 1000)" +ck "1b perfRate with 0 ns = 0" "$(eq "$(perfRate 5000 0)" 0)" +ck "1c perfGBmin 1000 MB/s = 60.0" "$(eq "$(perfGBmin 1000)" 60.0)" +ck "1d perfGBmin 125 MB/s = 7.5" "$(eq "$(perfGBmin 125)" 7.5)" + +# 2. The share: a non-imaging task is sent /dev/, and the stored images +# are its parent. A path without dev/ is left alone. +ck "2a dev/ share maps to its parent" "$(eq "$(perfImageRoot 10.0.0.1:/images/dev/)" 10.0.0.1:/images/)" +ck "2b no trailing slash too" "$(eq "$(perfImageRoot 10.0.0.1:/opt/fog/images/dev)" 10.0.0.1:/opt/fog/images/)" +ck "2c images root unchanged" "$(eq "$(perfImageRoot 10.0.0.1:/images/)" 10.0.0.1:/images/)" +ck "2d a dir merely ending in dev is not stripped" "$(eq "$(perfImageRoot 10.0.0.1:/srv/imagesdev/)" 10.0.0.1:/srv/imagesdev/)" + +# 3. The sample: the largest stored image file, never one under dev/ (a +# capture in progress), split zstd pieces included. +root="$tmp/images" +mkdir -p "$root/dev/001122334455" "$root/win10" "$root/win11" +head -c 300000 /dev/zero > "$root/dev/001122334455/d1p2.img.000" +head -c 1000 /dev/zero > "$root/win10/d1p2.img" +head -c 5000 /dev/zero > "$root/win11/d1p3.img.001" +head -c 9000 /dev/zero > "$root/win11/d1p3.mbr" +ck "3a largest stored image, dev/ skipped" "$(eq "$(perfPickSample "$root")" "$root/win11/d1p3.img.001")" +mkdir -p "$tmp/empty" +ck "3b no images gives nothing" "$(eq "$(perfPickSample "$tmp/empty")" "")" + +# 4. Format from magic bytes, using real compressor output. +head -c 4000000 /dev/urandom | base64 > "$tmp/plain" +zstd -q -c "$tmp/plain" > "$tmp/s.zst" +pigz -c "$tmp/plain" > "$tmp/s.gz" +ck "4a zstd detected" "$(eq "$(perfFormat "$tmp/s.zst")" zstd)" +ck "4b pigz/gzip detected" "$(eq "$(perfFormat "$tmp/s.gz")" gzip)" +ck "4c anything else is unknown" "$(eq "$(perfFormat "$tmp/plain")" unknown)" +ck "4d zstd uses the deploy's zstdmt -dc" "$(eq "$(perfDecompressor zstd)" "zstdmt -dc")" +ck "4e gzip uses the deploy's pigz -dc" "$(eq "$(perfDecompressor gzip)" "pigz -dc")" + +# 5. Decompression of a sample cut mid-stream still yields its output, and +# more output than input for compressible data. +for f in zst gz; do + head -c 1000000 "$tmp/s.$f" > "$tmp/cut.$f" + fmt=$([[ $f == zst ]] && echo zstd || echo gzip) + perfDecompressTest "$tmp/cut.$f" "$fmt" + ck "5a $fmt cut stream: input counted" "$(eq "$PERF_IN" 1000000)" + ck "5b $fmt cut stream: output > input" "$([[ $PERF_OUT -gt $PERF_IN ]] && echo 1 || echo 0)" + ck "5c $fmt cut stream: timed" "$([[ $PERF_NS -gt 0 ]] && echo 1 || echo 0)" +done +perfDecompressTest "$tmp/plain" unknown +ck "5d unknown format is refused" "$(eq "$?" 1)" + +# 6. Ratio and scaling: the network rate moves compressed bytes and is scaled +# to uncompressed, the unit the disk is measured in. +ck "6a ratio 2.5:1 is 250" "$(eq "$(perfRatio100 1000 2500)" 250)" +ck "6b missing counts give 100" "$(eq "$(perfRatio100 0 2500)" 100)" +ck "6c 100 MB/s compressed at 2.5:1 is 250" "$(eq "$(perfScale 100 250)" 250)" + +# 7. The bottleneck is the slowest MEASURED stage. +ck "7a network slowest" "$(eq "$(perfBottleneck 500 100 300)" network)" +ck "7b cpu slowest" "$(eq "$(perfBottleneck 500 400 300)" cpu)" +ck "7c unmeasured stages are ignored" "$(eq "$(perfBottleneck 500 0 "")" disk)" +ck "7d nothing measured names nothing" "$(eq "$(perfBottleneck 0 0 0)" "")" + +# 8. A read shorter than asked reports the bytes actually moved. +head -c 3145728 /dev/zero > "$tmp/three" +perfReadTest "$tmp/three" 8 0 +ck "8a short source: bytes actually read" "$(eq "$PERF_BYTES" 3145728)" +ck "8b short source: timed" "$([[ $PERF_NS -gt 0 ]] && echo 1 || echo 0)" + +# 9. The task itself: dispatched by bin/fog, executable, mounts read-only, +# reports before it completes the task. +grep -qE '^\s+perftest\)' "$OVERLAY/bin/fog" && grep -q 'fog.perftest' "$OVERLAY/bin/fog" +ck "9a bin/fog dispatches mode=perftest" "$(eq $? 0)" +ck "9b fog.perftest is executable" "$([[ -x $OVERLAY/bin/fog.perftest ]] && echo 1 || echo 0)" +grep -qE 'mount -t nfs -o ro,' "$OVERLAY/bin/fog.perftest" +ck "9c storage is mounted read-only" "$(eq $? 0)" +report=$(grep -n 'service/perftest.php' "$OVERLAY/bin/fog.perftest" | cut -d: -f1) +complete=$(grep -n 'fog.nonimgcomplete' "$OVERLAY/bin/fog.perftest" | cut -d: -f1) +ck "9d result is reported before the task completes" "$([[ -n $report && -n $complete && $report -lt $complete ]] && echo 1 || echo 0)" + +echo +echo "passed: $((checked - fails)) failed: $fails" +[[ $fails -eq 0 ]]