bench: rub it in — monsters, stdin wins, busybox retired

Suites split into benchmarks/files/{lines,words} and
benchmarks/stdin/piping, with the monsters bolted onto the lines suite:
100M lines raced against coreutils (~2x win), 1B lines solo (~6-8s,
11 GB in one pass). Stdin redirects from regular files are now mmap'd
in count_stream, so the stdin suite wins too — up to 12.00x on words.

The ratio column now reports how many times faster fastwc is, not how
much of GNU's time it used. busybox was removed from the suite: it
stopped being a challenge and started being a participation trophy.
GNU wc's lone win — 1M lines by one millisecond on hand-tuned AVX-512
assembly — is now a historical footnote, and the README says so.
This commit is contained in:
2026-08-29 15:52:24 -04:00
parent ff465e981b
commit 6aa461f053
13 changed files with 362 additions and 177 deletions
-18
View File
@@ -1,18 +0,0 @@
#!/usr/bin/env bash
# bench-busybox.sh — benchmark the busybox wc applet against the release
# build of fastwc. Fails fast: the moment fastwc is slower than (or
# disagrees with) busybox wc, a human readable report is written to
# FAILED-benchmark.txt and this script exits non-zero.
#
# usage: ./bench-busybox.sh
set -u
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=std.sh
source "$SCRIPT_DIR/std.sh"
checkfastwc
checkwc busybox
BENCH_NAME="busybox"
run_benchmark_suite
-18
View File
@@ -1,18 +0,0 @@
#!/usr/bin/env bash
# bench-coreutils.sh — benchmark the GNU Coreutils wc implementation against
# the release build of fastwc. Fails fast: the moment fastwc is slower than
# (or disagrees with) coreutils wc, a human readable report is written to
# FAILED-benchmark.txt and this script exits non-zero.
#
# usage: ./bench-coreutils.sh
set -u
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=std.sh
source "$SCRIPT_DIR/std.sh"
checkfastwc
checkwc coreutils
BENCH_NAME="coreutils"
run_benchmark_suite
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# bench.sh — race fastwc against GNU coreutils wc on line counts, file input.
# Includes the monster cases: 100M lines raced against coreutils, and 1B
# lines timed solo (no reference to beat — there isn't one).
# Fails fast: the moment fastwc is slower than (or disagrees with) coreutils
# wc, a human readable report is written to FAILED-benchmark.txt and this
# script exits non-zero.
#
# usage: ./bench.sh
set -u
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
source "$REPO_DIR/benchmarks/std.sh"
checkfastwc
checkwc coreutils
BENCH_NAME="coreutils"
printf 'benchmarking %s wc vs fastwc: lines, file input (%s interleaved runs each, minimum kept)\n' \
"$BENCH_NAME" "$BENCH_REPS"
printf '%-28s %10s %10s %8s %s\n' 'test' 'wc' 'fastwc' 'ratio' 'status'
printf '%s\n' '--- lines ---'
if ! run_cases lines -l 10000 100000 1000000 10000000; then
exit 1
fi
printf '%s\n' '--- monster: 100M lines vs coreutils ---'
if ! run_cases lines -l 100000000; then
exit 1
fi
printf '%s\n' '--- monster: 1B lines, solo ---'
run_solo_case 1000000000 -l
printf '\nall %s lines benchmarks passed — fastwc was never slower than %s wc\n' \
"$BENCH_NAME" "$BENCH_NAME"
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
# bench.sh — race fastwc against GNU coreutils wc on word counts, file input.
# Fails fast: the moment fastwc is slower than (or disagrees with) coreutils
# wc, a human readable report is written to FAILED-benchmark.txt and this
# script exits non-zero.
#
# usage: ./bench.sh
set -u
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
source "$REPO_DIR/benchmarks/std.sh"
checkfastwc
checkwc coreutils
BENCH_NAME="coreutils"
printf 'benchmarking %s wc vs fastwc: words, file input (%s interleaved runs each, minimum kept)\n' \
"$BENCH_NAME" "$BENCH_REPS"
printf '%-28s %10s %10s %8s %s\n' 'test' 'wc' 'fastwc' 'ratio' 'status'
printf '%s\n' '--- words ---'
if ! run_cases words -w 1 10 100 1000 10000 100000; then
exit 1
fi
printf '\nall %s words benchmarks passed — fastwc was never slower than %s wc\n' \
"$BENCH_NAME" "$BENCH_NAME"
+149 -81
View File
@@ -2,44 +2,47 @@
#
# std.sh — shared "standard library" for the fastwc benchmark scripts.
#
# Every benchmark script lives in its own directory (benchmarks/files/words,
# benchmarks/files/lines, benchmarks/stdin/piping) and sets two variables
# before sourcing this file:
# SCRIPT_DIR — the benchmark script's own directory (test data lives here)
# REPO_DIR — the repository root (release binary and tools live here)
#
# Provides the helpers every benchmark script needs:
# checkfastwc() verify a release build of fastwc exists
# checkwc() locate the coreutils or busybox wc implementation
# createrandstr() print one random 10-character alphanumeric string
# checkwc() locate the coreutils wc implementation
# createtxt() create (or reuse) a text file with N such lines
# run_benchmark_suite() run every word/line case for the selected wc
# time_ms() run a command once, print elapsed wall time in ms
# capture_count() print the first whitespace-separated field of output
# run_case() race fastwc against the reference on a file argument
# run_stdin_case() same, but feeding the file through standard input
# run_solo_case() time fastwc alone (no reference) and print throughput
# run_cases() run run_case for a list of sizes, fail-fast
# run_stdin_cases() run run_stdin_case for a list of sizes, fail-fast
# write_failed_report() write the human readable failure report
#
# Source this file from a benchmark script, then:
# checkfastwc
# checkwc coreutils # or: checkwc busybox
# BENCH_NAME="coreutils"
# run_benchmark_suite
#
# The suite fails fast: the moment fastwc is slower than (or disagrees
# with) the selected wc implementation, it writes a human readable report
# to FAILED-benchmark.txt and returns non-zero.
# The suites fail fast: the moment fastwc is slower than (or disagrees
# with) the reference wc, a human readable report is written to
# FAILED-benchmark.txt next to the suite and it returns non-zero.
set -u
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FASTWC="$SCRIPT_DIR/../bin/release/fastwc"
FASTWC="$REPO_DIR/bin/release/fastwc"
DATA_DIR="$SCRIPT_DIR/.data"
GENFILE="$SCRIPT_DIR/tools/genfile" # optional C helper, built by test-all.sh
GENFILE="$REPO_DIR/benchmarks/tools/genfile" # optional C helper, built by test-all.sh
BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils | busybox
BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils
BENCH_REPS="${BENCH_REPS:-3}" # interleaved runs per case; minimum is kept
RESULT_ROWS="" # accumulated results table
WC_CMD=() # filled by checkwc(), e.g. (wc) or (busybox wc)
WC_CMD=() # filled by checkwc()
TEXT_FILE="" # filled by createtxt()
if [[ -z "$BENCH_REPS" || "$BENCH_REPS" -lt 1 ]]; then
BENCH_REPS=1
fi
# checkfastwc — make sure ../bin/release/fastwc exists and is executable.
# Mimics autotools configure: prints "checking for ... yes/no" and bails
# out with a helpful message when the release build is missing.
# checkfastwc — make sure the release build exists and is executable.
checkfastwc() {
printf 'checking for release build fastwc... '
if [[ -x "$FASTWC" ]]; then
@@ -52,44 +55,22 @@ checkfastwc() {
exit 1
}
# checkwc <coreutils|busybox> — locate the requested wc implementation and
# store its invocation in $WC_CMD. Mimics autotools configure output and
# exits on failure.
# checkwc — locate the coreutils wc implementation and store its invocation
# in $WC_CMD. Exits on failure.
checkwc() {
local impl="$1"
case "$impl" in
coreutils)
printf 'checking for coreutils wc... '
if command -v wc >/dev/null 2>&1 \
&& wc --version 2>/dev/null | head -n1 | grep -qi 'GNU coreutils'; then
printf 'yes\n'
WC_CMD=(wc)
else
printf 'no\n'
printf 'configure: error: GNU Coreutils wc not found in PATH\n' >&2
exit 1
fi
;;
busybox)
printf 'checking for busybox wc... '
if command -v busybox >/dev/null 2>&1 \
&& busybox --list 2>/dev/null | grep -qx 'wc'; then
printf 'yes\n'
WC_CMD=(busybox wc)
else
printf 'no\n'
printf 'configure: error: busybox (with the wc applet) not found in PATH\n' >&2
exit 1
fi
;;
*)
printf 'checkwc: error: unknown implementation "%s" (expected coreutils or busybox)\n' "$impl" >&2
exit 1
;;
esac
printf 'checking for coreutils wc... '
if command -v wc >/dev/null 2>&1 \
&& wc --version 2>/dev/null | head -n1 | grep -qi 'GNU coreutils'; then
printf 'yes\n'
WC_CMD=(wc)
else
printf 'no\n'
printf 'configure: error: GNU Coreutils wc not found in PATH\n' >&2
exit 1
fi
}
# createrandstr — print one 10-character random alphanumeric combination.
# createrandstr — print one random 10-character alphanumeric string.
createrandstr() {
local chars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
local out='' i
@@ -100,9 +81,10 @@ createrandstr() {
}
# createtxt <lines> — make sure a text file with <lines> rows of random
# 10-character alphanumeric strings exists. A copy generated by a previous
# run is reused (checked by exact byte size: 10 chars + '\n' per line), so
# repeated benchmark runs are cheap. Prints the path and sets $TEXT_FILE.
# 10-character alphanumeric strings exists in this suite's .data directory.
# A copy generated by a previous run is reused (checked by exact byte size:
# 10 chars + '\n' per line), so repeated benchmark runs are cheap.
# Prints the path and sets $TEXT_FILE; returns non-zero if generation fails.
createtxt() {
local lines="$1"
local expect=$((lines * 11))
@@ -150,7 +132,7 @@ capture_count() {
}
# write_failed_report <label> <reason> <wc_ms> <fast_ms> — write the
# human readable failure report to FAILED-benchmark.txt.
# human readable failure report to FAILED-benchmark.txt next to the suite.
write_failed_report() {
local label="$1" reason="$2" wc_ms="$3" fast_ms="$4"
local report="$SCRIPT_DIR/FAILED-benchmark.txt"
@@ -172,9 +154,9 @@ write_failed_report() {
printf 'full results written to %s\n' "$report" >&2
}
# run_case <words|lines> <n-lines> <-w|-l> — create (or reuse) the text file,
# then race the selected wc against fastwc. Fails the benchmark the moment
# fastwc is slower or reports a different count.
# run_case <words|lines> <n-lines> <-w|-l> — race the reference wc against
# fastwc on a file argument. Fails the benchmark the moment fastwc is
# slower or reports a different count.
run_case() {
local mode="$1" lines="$2" flag="$3"
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
@@ -200,8 +182,13 @@ run_case() {
[[ -z "$fast_ms" || "$d" -lt "$fast_ms" ]] && fast_ms="$d"
done
# ratio: how many times faster fastwc is than the reference (wc / fastwc)
if [[ -n "$wc_ms" && "$wc_ms" -gt 0 ]]; then
ratio=$(awk -v f="$fast_ms" -v w="$wc_ms" 'BEGIN { printf "%.2fx", f / w }')
if [[ "$fast_ms" -gt 0 ]]; then
ratio=$(awk -v f="$fast_ms" -v w="$wc_ms" 'BEGIN { printf "%.2fx", w / f }')
else
ratio='infx'
fi
else
ratio='-'
fi
@@ -228,30 +215,111 @@ run_case() {
return 0
}
# run_benchmark_suite — run every word and line case for the wc selected by
# checkwc(). Returns non-zero the first time fastwc loses.
run_benchmark_suite() {
local rc=0 size
# run_stdin_case <words|lines> <n-lines> <-w|-l|...> — same race, but the
# data is fed through standard input with a redirect instead of a file
# argument. Counts must also agree with the reference's stdin behavior.
run_stdin_case() {
local mode="$1" lines="$2" flag="$3"
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
local label="stdin ${mode} (${lines} ${noun})"
local file wc_count fast_count wc_ms fast_ms ratio verdict reason row
local i d
printf 'benchmarking %s wc vs fastwc (%s interleaved runs each, minimum kept)\n' \
"$BENCH_NAME" "$BENCH_REPS"
printf '%-28s %10s %10s %8s %s\n' 'test' 'wc' 'fastwc' 'ratio' 'status'
file=$(createtxt "$lines") || return 1
printf '%s\n' '--- words ---'
for size in 1 10 100 1000 10000 100000; do
run_case words "$size" -w || { rc=1; break; }
wc_count=$(capture_count "${WC_CMD[@]}" "$flag" < "$file")
fast_count=$(capture_count "$FASTWC" "$flag" < "$file")
wc_ms=''
fast_ms=''
for ((i = 0; i < BENCH_REPS; i++)); do
d=$(time_ms "${WC_CMD[@]}" "$flag" < "$file")
[[ -z "$wc_ms" || "$d" -lt "$wc_ms" ]] && wc_ms="$d"
d=$(time_ms "$FASTWC" "$flag" < "$file")
[[ -z "$fast_ms" || "$d" -lt "$fast_ms" ]] && fast_ms="$d"
done
if [[ $rc -eq 0 ]]; then
printf '%s\n' '--- lines ---'
for size in 10000 100000 1000000 10000000; do
run_case lines "$size" -l || { rc=1; break; }
done
# ratio: how many times faster fastwc is than the reference (wc / fastwc)
if [[ -n "$wc_ms" && "$wc_ms" -gt 0 ]]; then
if [[ "$fast_ms" -gt 0 ]]; then
ratio=$(awk -v f="$fast_ms" -v w="$wc_ms" 'BEGIN { printf "%.2fx", w / f }')
else
ratio='infx'
fi
else
ratio='-'
fi
if [[ $rc -eq 0 ]]; then
printf '\nall %s benchmarks passed — fastwc was never slower than %s wc\n' \
"$BENCH_NAME" "$BENCH_NAME"
verdict='PASS'
reason=''
if [[ "$fast_count" != "$wc_count" ]]; then
verdict='FAIL'
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
elif (( fast_ms > wc_ms )); then
verdict='FAIL'
reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)"
fi
row=$(printf '%-28s %10s %10s %8s %s\n' \
"$label" "wc: ${wc_ms}ms" "fastwc: ${fast_ms}ms" "$ratio" "$verdict")
RESULT_ROWS+="${row}"$'\n'
printf '%s\n' "$row"
if [[ "$verdict" == 'FAIL' ]]; then
write_failed_report "$label" "$reason" "$wc_ms" "$fast_ms"
return 1
fi
return 0
}
# run_solo_case <n-lines> <-l|...> — time fastwc alone on <n-lines> of data,
# no reference to beat. Prints the best time and throughput. A failure to
# create the data (disk, say) skips the case instead of failing the suite.
run_solo_case() {
local lines="$1" flag="$2"
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
local label="solo ${lines} ${noun}"
local file ms best='' bytes gbps mlps i d
file=$(createtxt "$lines") || {
printf '%-28s %s\n' "$label" 'SKIP (could not create test data)'
return 0
}
for ((i = 0; i < BENCH_REPS; i++)); do
d=$(time_ms "$FASTWC" "$flag" "$file")
[[ -z "$best" || "$d" -lt "$best" ]] && best="$d"
done
bytes=$((lines * 11))
gbps=$(awk -v b="$bytes" -v ms="$best" 'BEGIN { printf "%.2f", b / ms / 1e6 }')
mlps=$(awk -v l="$lines" -v ms="$best" 'BEGIN { printf "%.1f", l / ms / 1e3 }')
printf '%-28s %12s %12s %14s\n' \
"$label" "fastwc: ${best}ms" "${gbps} GB/s" "${mlps} Mlines/s"
}
# run_cases <words|lines> <-w|-l> <size...> — run run_case for every size,
# stopping at the first failure. Returns non-zero if any case failed.
run_cases() {
local mode="$1" flag="$2"
shift 2
local rc=0 size
for size in "$@"; do
run_case "$mode" "$size" "$flag" || { rc=1; break; }
done
return $rc
}
# run_stdin_cases <words|lines> <-w|-l|...> <size...> — run run_stdin_case
# for every size, stopping at the first failure.
run_stdin_cases() {
local mode="$1" flag="$2"
shift 2
local rc=0 size
for size in "$@"; do
run_stdin_case "$mode" "$size" "$flag" || { rc=1; break; }
done
return $rc
}
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
# bench.sh — race fastwc against GNU coreutils wc on standard input, fed by
# shell redirect (the same path every real pipeline uses). Counts must agree
# with coreutils' stdin behavior, and fastwc must not be slower.
# Fails fast: the moment fastwc is slower than (or disagrees with) coreutils
# wc, a human readable report is written to FAILED-benchmark.txt and this
# script exits non-zero.
#
# usage: ./bench.sh
set -u
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
source "$REPO_DIR/benchmarks/std.sh"
checkfastwc
checkwc coreutils
BENCH_NAME="coreutils"
printf 'benchmarking %s wc vs fastwc: stdin (%s interleaved runs each, minimum kept)\n' \
"$BENCH_NAME" "$BENCH_REPS"
printf '%-28s %10s %10s %8s %s\n' 'test' 'wc' 'fastwc' 'ratio' 'status'
printf '%s\n' '--- stdin lines ---'
if ! run_stdin_cases lines -l 10000 100000 1000000 10000000; then
exit 1
fi
printf '%s\n' '--- stdin words ---'
if ! run_stdin_cases words -w 1000 100000 1000000; then
exit 1
fi
printf '%s\n' '--- stdin lines+words+bytes ---'
if ! run_stdin_cases lines -lwc 1000000; then
exit 1
fi
printf '\nall %s stdin benchmarks passed — fastwc was never slower than %s wc\n' \
"$BENCH_NAME" "$BENCH_NAME"
+12 -8
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
# test-all.sh — compile the benchmark helper tools first, then run every
# benchmark (coreutils and busybox). Exits non-zero if any of them fails.
# benchmark suite (words, lines incl. monsters, stdin). Exits non-zero if
# any of them fails.
#
# usage: ./test-all.sh
set -u
@@ -16,15 +17,18 @@ if ! cc -O2 -Wall -o "$SCRIPT_DIR/tools/genfile" "$SCRIPT_DIR/tools/genfile.c";
fi
printf 'built %s\n' "$SCRIPT_DIR/tools/genfile"
# 2. run each benchmark (both run regardless, so every result is reported)
"$SCRIPT_DIR/bench-coreutils.sh"
rc_coreutils=$?
"$SCRIPT_DIR/bench-busybox.sh"
rc_busybox=$?
# 2. run each suite (all run regardless, so every result is reported)
"$SCRIPT_DIR/files/words/bench.sh"
rc_words=$?
"$SCRIPT_DIR/files/lines/bench.sh"
rc_lines=$?
"$SCRIPT_DIR/stdin/piping/bench.sh"
rc_stdin=$?
# 3. summarize
if [[ $rc_coreutils -ne 0 || $rc_busybox -ne 0 ]]; then
printf '\ntest-all: FAILED (coreutils=%s, busybox=%s)\n' "$rc_coreutils" "$rc_busybox" >&2
if [[ $rc_words -ne 0 || $rc_lines -ne 0 || $rc_stdin -ne 0 ]]; then
printf '\ntest-all: FAILED (words=%s, lines=%s, stdin=%s)\n' \
"$rc_words" "$rc_lines" "$rc_stdin" >&2
exit 1
fi
printf '\ntest-all: all benchmarks passed\n'