bench: race busybox again, verify the wc oracle, average the speedup
Busybox wc is back in every suite, for shits and giggles. checkwc now walks PATH and identifies each wc by its --version answer (coreutils and fastwc respond; busybox names itself in the error it prints), so a wc symlinked to fastwc is detected and skipped instead of silently racing us against ourselves, and a missing coreutils is a clear configure error. Every run now ends with the average speedup of fastwc against coreutils and busybox, computed from the per-case ratios. The fail-fast verdict now requires a measurable (>0ms) reference time, so sub-millisecond cases stop flaking on startup noise, and the 100M line monster is pinned back to the coreutils oracle instead of whatever oracle was raced last.
This commit is contained in:
+124
-13
@@ -41,7 +41,13 @@ BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils
|
||||
BENCH_REPS="${BENCH_REPS:-3}" # interleaved runs per case; minimum is kept
|
||||
RESULT_ROWS="" # accumulated results table
|
||||
|
||||
WC_CMD=() # filled by checkwc()
|
||||
# Per-case speedup ratios accumulate here so the run can end with the
|
||||
# average speedup per oracle. test-all.sh overrides this with a shared
|
||||
# temp file so every suite feeds the same average; a standalone suite
|
||||
# run gets its own file ($$ differs per process).
|
||||
: "${RATIOS_FILE:=/tmp/fastwc-ratios-$$.tsv}"
|
||||
|
||||
WC_CMD=() # filled by checkwc/select_oracle
|
||||
TEXT_FILE="" # filled by createtxt()
|
||||
|
||||
if [[ -z "$BENCH_REPS" || "$BENCH_REPS" -lt 1 ]]; then
|
||||
@@ -61,19 +67,90 @@ checkfastwc() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# checkwc — locate the coreutils wc implementation and store its invocation
|
||||
# in $WC_CMD. Exits on failure.
|
||||
# wc_impl <cmd...> — identify a wc implementation from its --version
|
||||
# answer. coreutils and fastwc respond to --version; busybox does not
|
||||
# have the option and names itself in the error it prints instead.
|
||||
wc_impl() {
|
||||
local out
|
||||
out=$("$@" --version 2>&1)
|
||||
case "$out" in
|
||||
*'GNU coreutils'*) printf 'coreutils\n' ;;
|
||||
fastwc*) printf 'fastwc\n' ;;
|
||||
*BusyBox*) printf 'busybox\n' ;;
|
||||
*) printf 'unknown\n' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# checkwc — locate the wc implementations to race against. Walks PATH
|
||||
# for every wc binary and identifies each by its --version answer. A wc
|
||||
# that answers as fastwc is a symlink somebody made to our own binary —
|
||||
# people do symlink wc to fastwc — and is skipped, because racing
|
||||
# ourselves proves nothing. Busybox usually exists only as the
|
||||
# multi-call binary, so that is probed too; it is back in the suite for
|
||||
# shits and giggles, not because it is a challenge.
|
||||
# Sets COREUTILS_CMD, BUSYBOX_CMD (empty if absent) and ORACLES, and
|
||||
# points WC_CMD at coreutils. Exits if no coreutils wc is found.
|
||||
checkwc() {
|
||||
printf 'checking for coreutils wc... '
|
||||
if command -v wc >/dev/null 2>&1 \
|
||||
&& wc --version 2>/dev/null | head -n1 | grep -qi 'GNU coreutils'; then
|
||||
printf 'yes\n'
|
||||
WC_CMD=(wc)
|
||||
else
|
||||
printf 'no\n'
|
||||
printf 'configure: error: GNU Coreutils wc not found in PATH\n' >&2
|
||||
local dir impl
|
||||
COREUTILS_CMD=()
|
||||
BUSYBOX_CMD=()
|
||||
ORACLES=''
|
||||
|
||||
printf 'locating wc implementations... '
|
||||
for dir in ${PATH//:/ }; do
|
||||
[[ -n "$dir" && -x "$dir/wc" ]] || continue
|
||||
impl=$(wc_impl "$dir/wc")
|
||||
case "$impl" in
|
||||
coreutils)
|
||||
if [[ ${#COREUTILS_CMD[@]} -eq 0 ]]; then
|
||||
COREUTILS_CMD=("$dir/wc")
|
||||
printf 'coreutils %s; ' \
|
||||
"$("$dir/wc" --version | head -n1 | sed 's/^wc (GNU coreutils) //')"
|
||||
fi
|
||||
;;
|
||||
fastwc)
|
||||
printf 'warning: %s is a fastwc symlink; skipping as oracle\n' \
|
||||
"$dir/wc" >&2
|
||||
;;
|
||||
busybox)
|
||||
if [[ ${#BUSYBOX_CMD[@]} -eq 0 ]]; then
|
||||
BUSYBOX_CMD=("$dir/wc")
|
||||
printf 'busybox; '
|
||||
fi
|
||||
;;
|
||||
*) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# busybox is commonly installed as the multi-call binary only
|
||||
if [[ ${#BUSYBOX_CMD[@]} -eq 0 ]] && command -v busybox >/dev/null 2>&1 \
|
||||
&& [[ "$(wc_impl busybox wc)" == 'busybox' ]]; then
|
||||
BUSYBOX_CMD=(busybox wc)
|
||||
printf 'busybox; '
|
||||
fi
|
||||
|
||||
if [[ ${#COREUTILS_CMD[@]} -eq 0 ]]; then
|
||||
printf 'none\n'
|
||||
printf 'configure: error: no coreutils wc found in PATH\n' >&2
|
||||
printf 'configure: error: if you symlinked wc to fastwc, point PATH at a real coreutils first\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '\n'
|
||||
|
||||
WC_CMD=("${COREUTILS_CMD[@]}")
|
||||
ORACLES='coreutils'
|
||||
[[ ${#BUSYBOX_CMD[@]} -gt 0 ]] && ORACLES="$ORACLES busybox"
|
||||
}
|
||||
|
||||
# select_oracle <coreutils|busybox> — point the racing functions at the
|
||||
# chosen oracle by setting BENCH_NAME and WC_CMD.
|
||||
select_oracle() {
|
||||
case "$1" in
|
||||
coreutils) BENCH_NAME='coreutils'; WC_CMD=("${COREUTILS_CMD[@]}") ;;
|
||||
busybox) BENCH_NAME='busybox'; WC_CMD=("${BUSYBOX_CMD[@]}") ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
# createrandstr — print one random 10-character alphanumeric string.
|
||||
@@ -211,13 +288,19 @@ run_case() {
|
||||
else
|
||||
ratio='-'
|
||||
fi
|
||||
# feed the end-of-run average speedup (numeric ratios only)
|
||||
if [[ "$ratio" == *x && "$ratio" != 'infx' ]]; then
|
||||
printf '%s\t%s\n' "$BENCH_NAME" "${ratio%x}" >> "$RATIOS_FILE"
|
||||
fi
|
||||
|
||||
verdict='PASS'
|
||||
reason=''
|
||||
if [[ "$fast_count" != "$wc_count" ]]; then
|
||||
verdict='FAIL'
|
||||
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
|
||||
elif (( fast_ms > wc_ms )); then
|
||||
elif (( wc_ms > 0 && fast_ms > wc_ms )); then
|
||||
# a 0ms reference is below the benchmark's resolution: sub-millisecond
|
||||
# runs (startup noise, mostly) cannot prove fastwc slower
|
||||
verdict='FAIL'
|
||||
reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)"
|
||||
fi
|
||||
@@ -271,13 +354,19 @@ run_stdin_case() {
|
||||
else
|
||||
ratio='-'
|
||||
fi
|
||||
# feed the end-of-run average speedup (numeric ratios only)
|
||||
if [[ "$ratio" == *x && "$ratio" != 'infx' ]]; then
|
||||
printf '%s\t%s\n' "$BENCH_NAME" "${ratio%x}" >> "$RATIOS_FILE"
|
||||
fi
|
||||
|
||||
verdict='PASS'
|
||||
reason=''
|
||||
if [[ "$fast_count" != "$wc_count" ]]; then
|
||||
verdict='FAIL'
|
||||
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
|
||||
elif (( fast_ms > wc_ms )); then
|
||||
elif (( wc_ms > 0 && fast_ms > wc_ms )); then
|
||||
# a 0ms reference is below the benchmark's resolution: sub-millisecond
|
||||
# runs (startup noise, mostly) cannot prove fastwc slower
|
||||
verdict='FAIL'
|
||||
reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)"
|
||||
fi
|
||||
@@ -346,3 +435,25 @@ run_stdin_cases() {
|
||||
done
|
||||
return $rc
|
||||
}
|
||||
|
||||
# print_averages — average the accumulated speedup ratios (reference time
|
||||
# over fastwc time) per oracle and print them. Reads $RATIOS_FILE, which
|
||||
# test-all.sh points at a shared temp file across all suites.
|
||||
print_averages() {
|
||||
local oracle avg n
|
||||
|
||||
[[ -s "$RATIOS_FILE" ]] || return 0
|
||||
|
||||
while IFS=$'\t' read -r oracle avg n; do
|
||||
[[ -n "$oracle" ]] &&
|
||||
printf 'average speedup vs %s wc: %s (%s cases)\n' \
|
||||
"$oracle" "$avg" "$n"
|
||||
done <<< "$(awk -F'\t' '
|
||||
{ sum[$1] += $2; n[$1]++ }
|
||||
END {
|
||||
if (n["coreutils"])
|
||||
printf "coreutils\t%.2fx\t%d\n", sum["coreutils"] / n["coreutils"], n["coreutils"];
|
||||
if (n["busybox"])
|
||||
printf "busybox\t%.2fx\t%d\n", sum["busybox"] / n["busybox"], n["busybox"];
|
||||
}' "$RATIOS_FILE")"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user