huntedbytheirs
6aa461f053
Suites split into benchmarks/files/{lines,words} and
benchmarks/stdin/piping, with the monsters bolted onto the lines suite:
100M lines raced against coreutils (~2x win), 1B lines solo (~6-8s,
11 GB in one pass). Stdin redirects from regular files are now mmap'd
in count_stream, so the stdin suite wins too — up to 12.00x on words.
The ratio column now reports how many times faster fastwc is, not how
much of GNU's time it used. busybox was removed from the suite: it
stopped being a challenge and started being a participation trophy.
GNU wc's lone win — 1M lines by one millisecond on hand-tuned AVX-512
assembly — is now a historical footnote, and the README says so.
326 lines
11 KiB
Bash
Executable File
326 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# std.sh — shared "standard library" for the fastwc benchmark scripts.
|
|
#
|
|
# Every benchmark script lives in its own directory (benchmarks/files/words,
|
|
# benchmarks/files/lines, benchmarks/stdin/piping) and sets two variables
|
|
# before sourcing this file:
|
|
# SCRIPT_DIR — the benchmark script's own directory (test data lives here)
|
|
# REPO_DIR — the repository root (release binary and tools live here)
|
|
#
|
|
# Provides the helpers every benchmark script needs:
|
|
# checkfastwc() verify a release build of fastwc exists
|
|
# checkwc() locate the coreutils wc implementation
|
|
# createtxt() create (or reuse) a text file with N such lines
|
|
# time_ms() run a command once, print elapsed wall time in ms
|
|
# capture_count() print the first whitespace-separated field of output
|
|
# run_case() race fastwc against the reference on a file argument
|
|
# run_stdin_case() same, but feeding the file through standard input
|
|
# run_solo_case() time fastwc alone (no reference) and print throughput
|
|
# run_cases() run run_case for a list of sizes, fail-fast
|
|
# run_stdin_cases() run run_stdin_case for a list of sizes, fail-fast
|
|
# write_failed_report() write the human readable failure report
|
|
#
|
|
# The suites fail fast: the moment fastwc is slower than (or disagrees
|
|
# with) the reference wc, a human readable report is written to
|
|
# FAILED-benchmark.txt next to the suite and it returns non-zero.
|
|
|
|
set -u
|
|
|
|
FASTWC="$REPO_DIR/bin/release/fastwc"
|
|
DATA_DIR="$SCRIPT_DIR/.data"
|
|
GENFILE="$REPO_DIR/benchmarks/tools/genfile" # optional C helper, built by test-all.sh
|
|
|
|
BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils
|
|
BENCH_REPS="${BENCH_REPS:-3}" # interleaved runs per case; minimum is kept
|
|
RESULT_ROWS="" # accumulated results table
|
|
|
|
WC_CMD=() # filled by checkwc()
|
|
TEXT_FILE="" # filled by createtxt()
|
|
|
|
if [[ -z "$BENCH_REPS" || "$BENCH_REPS" -lt 1 ]]; then
|
|
BENCH_REPS=1
|
|
fi
|
|
|
|
# checkfastwc — make sure the release build exists and is executable.
|
|
checkfastwc() {
|
|
printf 'checking for release build fastwc... '
|
|
if [[ -x "$FASTWC" ]]; then
|
|
printf 'yes\n'
|
|
return 0
|
|
fi
|
|
printf 'no\n'
|
|
printf 'configure: error: no release build of fastwc found at %s\n' "$FASTWC" >&2
|
|
printf 'configure: error: run "make release" first to generate one\n' >&2
|
|
exit 1
|
|
}
|
|
|
|
# checkwc — locate the coreutils wc implementation and store its invocation
|
|
# in $WC_CMD. Exits on failure.
|
|
checkwc() {
|
|
printf 'checking for coreutils wc... '
|
|
if command -v wc >/dev/null 2>&1 \
|
|
&& wc --version 2>/dev/null | head -n1 | grep -qi 'GNU coreutils'; then
|
|
printf 'yes\n'
|
|
WC_CMD=(wc)
|
|
else
|
|
printf 'no\n'
|
|
printf 'configure: error: GNU Coreutils wc not found in PATH\n' >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# createrandstr — print one random 10-character alphanumeric string.
|
|
createrandstr() {
|
|
local chars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
|
|
local out='' i
|
|
for ((i = 0; i < 10; i++)); do
|
|
out+="${chars:$((RANDOM % ${#chars})):1}"
|
|
done
|
|
printf '%s\n' "$out"
|
|
}
|
|
|
|
# createtxt <lines> — make sure a text file with <lines> rows of random
|
|
# 10-character alphanumeric strings exists in this suite's .data directory.
|
|
# A copy generated by a previous run is reused (checked by exact byte size:
|
|
# 10 chars + '\n' per line), so repeated benchmark runs are cheap.
|
|
# Prints the path and sets $TEXT_FILE; returns non-zero if generation fails.
|
|
createtxt() {
|
|
local lines="$1"
|
|
local expect=$((lines * 11))
|
|
local have=0
|
|
|
|
TEXT_FILE="$DATA_DIR/words-$lines.txt"
|
|
|
|
if [[ -f "$TEXT_FILE" ]]; then
|
|
have=$(stat -c '%s' "$TEXT_FILE" 2>/dev/null || printf '0')
|
|
fi
|
|
|
|
if [[ "$have" -ne "$expect" ]]; then
|
|
mkdir -p "$DATA_DIR"
|
|
if [[ -x "$GENFILE" ]]; then
|
|
"$GENFILE" "$lines" > "$TEXT_FILE" || {
|
|
printf 'createtxt: error: failed to generate %s\n' "$TEXT_FILE" >&2
|
|
return 1
|
|
}
|
|
else
|
|
printf 'createtxt: warning: %s not built, using slow shell fallback\n' "$GENFILE" >&2
|
|
printf 'createtxt: warning: run ./test-all.sh to build the helper tools\n' >&2
|
|
: > "$TEXT_FILE"
|
|
for ((i = 0; i < lines; i++)); do
|
|
createrandstr >> "$TEXT_FILE"
|
|
done
|
|
fi
|
|
fi
|
|
|
|
printf '%s\n' "$TEXT_FILE"
|
|
}
|
|
|
|
# time_ms <cmd...> — run a command once and print elapsed wall time in ms.
|
|
time_ms() {
|
|
local s e
|
|
s=$(date +%s%N)
|
|
"$@" >/dev/null 2>&1
|
|
e=$(date +%s%N)
|
|
printf '%s\n' "$(( (e - s) / 1000000 ))"
|
|
}
|
|
|
|
# capture_count <cmd...> — print the first whitespace-separated field of a
|
|
# command's output, i.e. the count reported by `wc -w/-l` or `fastwc -w/-l`.
|
|
capture_count() {
|
|
"$@" 2>/dev/null | awk 'NR == 1 { print $1 }'
|
|
}
|
|
|
|
# write_failed_report <label> <reason> <wc_ms> <fast_ms> — write the
|
|
# human readable failure report to FAILED-benchmark.txt next to the suite.
|
|
write_failed_report() {
|
|
local label="$1" reason="$2" wc_ms="$3" fast_ms="$4"
|
|
local report="$SCRIPT_DIR/FAILED-benchmark.txt"
|
|
|
|
{
|
|
printf 'fastwc benchmark FAILED\n'
|
|
printf '=======================\n'
|
|
printf 'implementation : %s wc\n' "$BENCH_NAME"
|
|
printf 'failed test : %s\n' "$label"
|
|
printf 'failure : %s\n' "$reason"
|
|
printf '\nresults\n'
|
|
printf '%s\n' '-------'
|
|
printf '%-28s %10s %10s %8s %s\n' 'test' 'wc' 'fastwc' 'ratio' 'status'
|
|
printf '%s' "$RESULT_ROWS"
|
|
printf '\nfastwc must never be slower than %s wc — benchmark aborted.\n' "$BENCH_NAME"
|
|
} > "$report"
|
|
|
|
printf '\nbenchmark FAILED (%s): %s\n' "$BENCH_NAME" "$reason" >&2
|
|
printf 'full results written to %s\n' "$report" >&2
|
|
}
|
|
|
|
# run_case <words|lines> <n-lines> <-w|-l> — race the reference wc against
|
|
# fastwc on a file argument. Fails the benchmark the moment fastwc is
|
|
# slower or reports a different count.
|
|
run_case() {
|
|
local mode="$1" lines="$2" flag="$3"
|
|
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
|
|
local label="${mode} (${lines} ${noun})"
|
|
local file wc_count fast_count wc_ms fast_ms ratio verdict reason row
|
|
local i d
|
|
|
|
file=$(createtxt "$lines") || return 1
|
|
|
|
# correctness: fastwc must report the same count as the reference wc
|
|
wc_count=$(capture_count "${WC_CMD[@]}" "$flag" "$file")
|
|
fast_count=$(capture_count "$FASTWC" "$flag" "$file")
|
|
|
|
# speed: interleaved timing so both commands see identical cache warmth;
|
|
# keep the minimum of $BENCH_REPS runs each to reduce noise
|
|
wc_ms=''
|
|
fast_ms=''
|
|
for ((i = 0; i < BENCH_REPS; i++)); do
|
|
d=$(time_ms "${WC_CMD[@]}" "$flag" "$file")
|
|
[[ -z "$wc_ms" || "$d" -lt "$wc_ms" ]] && wc_ms="$d"
|
|
|
|
d=$(time_ms "$FASTWC" "$flag" "$file")
|
|
[[ -z "$fast_ms" || "$d" -lt "$fast_ms" ]] && fast_ms="$d"
|
|
done
|
|
|
|
# ratio: how many times faster fastwc is than the reference (wc / fastwc)
|
|
if [[ -n "$wc_ms" && "$wc_ms" -gt 0 ]]; then
|
|
if [[ "$fast_ms" -gt 0 ]]; then
|
|
ratio=$(awk -v f="$fast_ms" -v w="$wc_ms" 'BEGIN { printf "%.2fx", w / f }')
|
|
else
|
|
ratio='infx'
|
|
fi
|
|
else
|
|
ratio='-'
|
|
fi
|
|
|
|
verdict='PASS'
|
|
reason=''
|
|
if [[ "$fast_count" != "$wc_count" ]]; then
|
|
verdict='FAIL'
|
|
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
|
|
elif (( fast_ms > wc_ms )); then
|
|
verdict='FAIL'
|
|
reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)"
|
|
fi
|
|
|
|
row=$(printf '%-28s %10s %10s %8s %s\n' \
|
|
"$label" "wc: ${wc_ms}ms" "fastwc: ${fast_ms}ms" "$ratio" "$verdict")
|
|
RESULT_ROWS+="${row}"$'\n'
|
|
printf '%s\n' "$row"
|
|
|
|
if [[ "$verdict" == 'FAIL' ]]; then
|
|
write_failed_report "$label" "$reason" "$wc_ms" "$fast_ms"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# run_stdin_case <words|lines> <n-lines> <-w|-l|...> — same race, but the
|
|
# data is fed through standard input with a redirect instead of a file
|
|
# argument. Counts must also agree with the reference's stdin behavior.
|
|
run_stdin_case() {
|
|
local mode="$1" lines="$2" flag="$3"
|
|
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
|
|
local label="stdin ${mode} (${lines} ${noun})"
|
|
local file wc_count fast_count wc_ms fast_ms ratio verdict reason row
|
|
local i d
|
|
|
|
file=$(createtxt "$lines") || return 1
|
|
|
|
wc_count=$(capture_count "${WC_CMD[@]}" "$flag" < "$file")
|
|
fast_count=$(capture_count "$FASTWC" "$flag" < "$file")
|
|
|
|
wc_ms=''
|
|
fast_ms=''
|
|
for ((i = 0; i < BENCH_REPS; i++)); do
|
|
d=$(time_ms "${WC_CMD[@]}" "$flag" < "$file")
|
|
[[ -z "$wc_ms" || "$d" -lt "$wc_ms" ]] && wc_ms="$d"
|
|
|
|
d=$(time_ms "$FASTWC" "$flag" < "$file")
|
|
[[ -z "$fast_ms" || "$d" -lt "$fast_ms" ]] && fast_ms="$d"
|
|
done
|
|
|
|
# ratio: how many times faster fastwc is than the reference (wc / fastwc)
|
|
if [[ -n "$wc_ms" && "$wc_ms" -gt 0 ]]; then
|
|
if [[ "$fast_ms" -gt 0 ]]; then
|
|
ratio=$(awk -v f="$fast_ms" -v w="$wc_ms" 'BEGIN { printf "%.2fx", w / f }')
|
|
else
|
|
ratio='infx'
|
|
fi
|
|
else
|
|
ratio='-'
|
|
fi
|
|
|
|
verdict='PASS'
|
|
reason=''
|
|
if [[ "$fast_count" != "$wc_count" ]]; then
|
|
verdict='FAIL'
|
|
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
|
|
elif (( fast_ms > wc_ms )); then
|
|
verdict='FAIL'
|
|
reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)"
|
|
fi
|
|
|
|
row=$(printf '%-28s %10s %10s %8s %s\n' \
|
|
"$label" "wc: ${wc_ms}ms" "fastwc: ${fast_ms}ms" "$ratio" "$verdict")
|
|
RESULT_ROWS+="${row}"$'\n'
|
|
printf '%s\n' "$row"
|
|
|
|
if [[ "$verdict" == 'FAIL' ]]; then
|
|
write_failed_report "$label" "$reason" "$wc_ms" "$fast_ms"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# run_solo_case <n-lines> <-l|...> — time fastwc alone on <n-lines> of data,
|
|
# no reference to beat. Prints the best time and throughput. A failure to
|
|
# create the data (disk, say) skips the case instead of failing the suite.
|
|
run_solo_case() {
|
|
local lines="$1" flag="$2"
|
|
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
|
|
local label="solo ${lines} ${noun}"
|
|
local file ms best='' bytes gbps mlps i d
|
|
|
|
file=$(createtxt "$lines") || {
|
|
printf '%-28s %s\n' "$label" 'SKIP (could not create test data)'
|
|
return 0
|
|
}
|
|
|
|
for ((i = 0; i < BENCH_REPS; i++)); do
|
|
d=$(time_ms "$FASTWC" "$flag" "$file")
|
|
[[ -z "$best" || "$d" -lt "$best" ]] && best="$d"
|
|
done
|
|
|
|
bytes=$((lines * 11))
|
|
gbps=$(awk -v b="$bytes" -v ms="$best" 'BEGIN { printf "%.2f", b / ms / 1e6 }')
|
|
mlps=$(awk -v l="$lines" -v ms="$best" 'BEGIN { printf "%.1f", l / ms / 1e3 }')
|
|
|
|
printf '%-28s %12s %12s %14s\n' \
|
|
"$label" "fastwc: ${best}ms" "${gbps} GB/s" "${mlps} Mlines/s"
|
|
}
|
|
|
|
# run_cases <words|lines> <-w|-l> <size...> — run run_case for every size,
|
|
# stopping at the first failure. Returns non-zero if any case failed.
|
|
run_cases() {
|
|
local mode="$1" flag="$2"
|
|
shift 2
|
|
local rc=0 size
|
|
for size in "$@"; do
|
|
run_case "$mode" "$size" "$flag" || { rc=1; break; }
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
# run_stdin_cases <words|lines> <-w|-l|...> <size...> — run run_stdin_case
|
|
# for every size, stopping at the first failure.
|
|
run_stdin_cases() {
|
|
local mode="$1" flag="$2"
|
|
shift 2
|
|
local rc=0 size
|
|
for size in "$@"; do
|
|
run_stdin_case "$mode" "$size" "$flag" || { rc=1; break; }
|
|
done
|
|
return $rc
|
|
}
|