diff --git a/README.md b/README.md index 23d0446..25280bd 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,10 @@ GNU wc's lone win — 1M lines, by a single millisecond, on hand-tuned AVX-512 assembly — is a historical footnote now. We closed it, then the 10M case, then the 100M monster, and then we added a 1B-line race with no opponent, because beating GNU wc at this point is just cruelty to -software. Busybox was removed from the suite for the same reason: it -stopped being a challenge and started being a participation trophy. +software. Busybox is back in the suite too — for shits and giggles. It +stopped being a challenge years ago, but watching it count is the +closest thing this project has to a hobby. Every run ends with the +average speedup against each oracle, so the cruelty is quantified. ## Why @@ -99,13 +101,16 @@ make bench # build release + run every suite The suites live under `benchmarks/files/{lines,words}` (file input) and `benchmarks/stdin/piping` (standard input), and fail fast the moment -fastwc loses a single case. The lines suite also carries the monsters: -100M lines raced against coreutils, and 1B lines timed solo — there is -no reference for that one; we are the reference now. The suites -interleave runs so both commands see identical cache warmth, keep the -minimum, and fail the moment fastwc loses a single case. GNU `wc` is -used as an oracle the same way you'd use a broken clock: occasionally -it's right, and it's the only one around. +fastwc loses a single case. Every suite races against GNU coreutils wc +and, when it is installed, busybox wc. The benchmark locates the real +oracles by their `--version` answer — a `wc` that answers as fastwc is a +symlink somebody made, and is skipped rather than raced against itself. +The lines suite also carries the monsters: 100M lines raced against +coreutils, and 1B lines timed solo — there is no reference for that +one; we are the reference now. The suites interleave runs so both +commands see identical cache warmth, keep the minimum, and fail the +moment fastwc loses a single case. When the run ends, the average +speedup of fastwc against each oracle is printed. ## Development diff --git a/benchmarks/files/lines/bench.sh b/benchmarks/files/lines/bench.sh index 41d156f..33fe1fd 100755 --- a/benchmarks/files/lines/bench.sh +++ b/benchmarks/files/lines/bench.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # bench.sh — race fastwc against GNU coreutils wc on line counts, file input. -# Includes the monster cases: 100M lines raced against coreutils, and 1B -# lines timed solo (no reference to beat — there isn't one). -# Fails fast: the moment fastwc is slower than (or disagrees with) coreutils -# wc, a human readable report is written to FAILED-benchmark.txt and this -# script exits non-zero. +# Busybox wc is raced too, for shits and giggles. Includes the monster +# cases: 100M lines raced against coreutils (busybox through 1.1 GB is a +# spectator sport, not a benchmark), and 1B lines timed solo (no +# reference to beat — there isn't one). +# Fails fast: the moment fastwc is slower than (or disagrees with) an +# oracle wc, a human readable report is written to FAILED-benchmark.txt +# and this script exits non-zero. # # usage: ./bench.sh set -u @@ -15,17 +17,26 @@ source "$REPO_DIR/benchmarks/std.sh" checkfastwc checkwc -BENCH_NAME="coreutils" -printf 'benchmarking %s wc vs fastwc: lines, file input (%s interleaved runs each, minimum kept)\n' \ - "$BENCH_NAME" "$BENCH_REPS" -printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' +for oracle in $ORACLES; do + select_oracle "$oracle" || exit 1 -printf '%s\n' '--- lines ---' -if ! run_cases lines -l 10000 100000 1000000 10000000; then - exit 1 -fi + printf 'benchmarking %s wc vs fastwc: lines, file input (%s interleaved runs each, minimum kept)\n' \ + "$BENCH_NAME" "$BENCH_REPS" + printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' + printf '%s\n' '--- lines ---' + if ! run_cases lines -l 10000 100000 1000000 10000000; then + exit 1 + fi + + printf '\nall %s lines benchmarks passed — fastwc was never slower than %s wc\n' \ + "$BENCH_NAME" "$BENCH_NAME" +done + +# The monsters stay coreutils-only (busybox through 1.1 GB is a spectator +# sport, not a benchmark), so switch back from the last oracle raced. +select_oracle coreutils || exit 1 printf '%s\n' '--- monster: 100M lines vs coreutils ---' if ! run_cases lines -l 100000000; then exit 1 @@ -33,6 +44,3 @@ fi printf '%s\n' '--- monster: 1B lines, solo ---' run_solo_case 1000000000 -l - -printf '\nall %s lines benchmarks passed — fastwc was never slower than %s wc\n' \ - "$BENCH_NAME" "$BENCH_NAME" diff --git a/benchmarks/files/words/bench.sh b/benchmarks/files/words/bench.sh index 8ea852b..0b30649 100755 --- a/benchmarks/files/words/bench.sh +++ b/benchmarks/files/words/bench.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # bench.sh — race fastwc against GNU coreutils wc on word counts, file input. -# Fails fast: the moment fastwc is slower than (or disagrees with) coreutils -# wc, a human readable report is written to FAILED-benchmark.txt and this -# script exits non-zero. +# Busybox wc is raced too, for shits and giggles. +# Fails fast: the moment fastwc is slower than (or disagrees with) an +# oracle wc, a human readable report is written to FAILED-benchmark.txt +# and this script exits non-zero. # # usage: ./bench.sh set -u @@ -13,16 +14,19 @@ source "$REPO_DIR/benchmarks/std.sh" checkfastwc checkwc -BENCH_NAME="coreutils" -printf 'benchmarking %s wc vs fastwc: words, file input (%s interleaved runs each, minimum kept)\n' \ - "$BENCH_NAME" "$BENCH_REPS" -printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' +for oracle in $ORACLES; do + select_oracle "$oracle" || exit 1 -printf '%s\n' '--- words ---' -if ! run_cases words -w 1 10 100 1000 10000 100000; then - exit 1 -fi + printf 'benchmarking %s wc vs fastwc: words, file input (%s interleaved runs each, minimum kept)\n' \ + "$BENCH_NAME" "$BENCH_REPS" + printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' -printf '\nall %s words benchmarks passed — fastwc was never slower than %s wc\n' \ - "$BENCH_NAME" "$BENCH_NAME" + printf '%s\n' '--- words ---' + if ! run_cases words -w 1 10 100 1000 10000 100000; then + exit 1 + fi + + printf '\nall %s words benchmarks passed — fastwc was never slower than %s wc\n' \ + "$BENCH_NAME" "$BENCH_NAME" +done diff --git a/benchmarks/std.sh b/benchmarks/std.sh index 3c8c6b5..1fa4d96 100755 --- a/benchmarks/std.sh +++ b/benchmarks/std.sh @@ -41,7 +41,13 @@ BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils BENCH_REPS="${BENCH_REPS:-3}" # interleaved runs per case; minimum is kept RESULT_ROWS="" # accumulated results table -WC_CMD=() # filled by checkwc() +# Per-case speedup ratios accumulate here so the run can end with the +# average speedup per oracle. test-all.sh overrides this with a shared +# temp file so every suite feeds the same average; a standalone suite +# run gets its own file ($$ differs per process). +: "${RATIOS_FILE:=/tmp/fastwc-ratios-$$.tsv}" + +WC_CMD=() # filled by checkwc/select_oracle TEXT_FILE="" # filled by createtxt() if [[ -z "$BENCH_REPS" || "$BENCH_REPS" -lt 1 ]]; then @@ -61,19 +67,90 @@ checkfastwc() { exit 1 } -# checkwc — locate the coreutils wc implementation and store its invocation -# in $WC_CMD. Exits on failure. +# wc_impl — identify a wc implementation from its --version +# answer. coreutils and fastwc respond to --version; busybox does not +# have the option and names itself in the error it prints instead. +wc_impl() { + local out + out=$("$@" --version 2>&1) + case "$out" in + *'GNU coreutils'*) printf 'coreutils\n' ;; + fastwc*) printf 'fastwc\n' ;; + *BusyBox*) printf 'busybox\n' ;; + *) printf 'unknown\n' ;; + esac +} + +# checkwc — locate the wc implementations to race against. Walks PATH +# for every wc binary and identifies each by its --version answer. A wc +# that answers as fastwc is a symlink somebody made to our own binary — +# people do symlink wc to fastwc — and is skipped, because racing +# ourselves proves nothing. Busybox usually exists only as the +# multi-call binary, so that is probed too; it is back in the suite for +# shits and giggles, not because it is a challenge. +# Sets COREUTILS_CMD, BUSYBOX_CMD (empty if absent) and ORACLES, and +# points WC_CMD at coreutils. Exits if no coreutils wc is found. checkwc() { - printf 'checking for coreutils wc... ' - if command -v wc >/dev/null 2>&1 \ - && wc --version 2>/dev/null | head -n1 | grep -qi 'GNU coreutils'; then - printf 'yes\n' - WC_CMD=(wc) - else - printf 'no\n' - printf 'configure: error: GNU Coreutils wc not found in PATH\n' >&2 + local dir impl + COREUTILS_CMD=() + BUSYBOX_CMD=() + ORACLES='' + + printf 'locating wc implementations... ' + for dir in ${PATH//:/ }; do + [[ -n "$dir" && -x "$dir/wc" ]] || continue + impl=$(wc_impl "$dir/wc") + case "$impl" in + coreutils) + if [[ ${#COREUTILS_CMD[@]} -eq 0 ]]; then + COREUTILS_CMD=("$dir/wc") + printf 'coreutils %s; ' \ + "$("$dir/wc" --version | head -n1 | sed 's/^wc (GNU coreutils) //')" + fi + ;; + fastwc) + printf 'warning: %s is a fastwc symlink; skipping as oracle\n' \ + "$dir/wc" >&2 + ;; + busybox) + if [[ ${#BUSYBOX_CMD[@]} -eq 0 ]]; then + BUSYBOX_CMD=("$dir/wc") + printf 'busybox; ' + fi + ;; + *) ;; + esac + done + + # busybox is commonly installed as the multi-call binary only + if [[ ${#BUSYBOX_CMD[@]} -eq 0 ]] && command -v busybox >/dev/null 2>&1 \ + && [[ "$(wc_impl busybox wc)" == 'busybox' ]]; then + BUSYBOX_CMD=(busybox wc) + printf 'busybox; ' + fi + + if [[ ${#COREUTILS_CMD[@]} -eq 0 ]]; then + printf 'none\n' + printf 'configure: error: no coreutils wc found in PATH\n' >&2 + printf 'configure: error: if you symlinked wc to fastwc, point PATH at a real coreutils first\n' >&2 exit 1 fi + printf '\n' + + WC_CMD=("${COREUTILS_CMD[@]}") + ORACLES='coreutils' + [[ ${#BUSYBOX_CMD[@]} -gt 0 ]] && ORACLES="$ORACLES busybox" +} + +# select_oracle — point the racing functions at the +# chosen oracle by setting BENCH_NAME and WC_CMD. +select_oracle() { + case "$1" in + coreutils) BENCH_NAME='coreutils'; WC_CMD=("${COREUTILS_CMD[@]}") ;; + busybox) BENCH_NAME='busybox'; WC_CMD=("${BUSYBOX_CMD[@]}") ;; + *) return 1 ;; + esac + return 0 } # createrandstr — print one random 10-character alphanumeric string. @@ -211,13 +288,19 @@ run_case() { else ratio='-' fi + # feed the end-of-run average speedup (numeric ratios only) + if [[ "$ratio" == *x && "$ratio" != 'infx' ]]; then + printf '%s\t%s\n' "$BENCH_NAME" "${ratio%x}" >> "$RATIOS_FILE" + fi verdict='PASS' reason='' if [[ "$fast_count" != "$wc_count" ]]; then verdict='FAIL' reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})" - elif (( fast_ms > wc_ms )); then + elif (( wc_ms > 0 && fast_ms > wc_ms )); then + # a 0ms reference is below the benchmark's resolution: sub-millisecond + # runs (startup noise, mostly) cannot prove fastwc slower verdict='FAIL' reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)" fi @@ -271,13 +354,19 @@ run_stdin_case() { else ratio='-' fi + # feed the end-of-run average speedup (numeric ratios only) + if [[ "$ratio" == *x && "$ratio" != 'infx' ]]; then + printf '%s\t%s\n' "$BENCH_NAME" "${ratio%x}" >> "$RATIOS_FILE" + fi verdict='PASS' reason='' if [[ "$fast_count" != "$wc_count" ]]; then verdict='FAIL' reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})" - elif (( fast_ms > wc_ms )); then + elif (( wc_ms > 0 && fast_ms > wc_ms )); then + # a 0ms reference is below the benchmark's resolution: sub-millisecond + # runs (startup noise, mostly) cannot prove fastwc slower verdict='FAIL' reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)" fi @@ -346,3 +435,25 @@ run_stdin_cases() { done return $rc } + +# print_averages — average the accumulated speedup ratios (reference time +# over fastwc time) per oracle and print them. Reads $RATIOS_FILE, which +# test-all.sh points at a shared temp file across all suites. +print_averages() { + local oracle avg n + + [[ -s "$RATIOS_FILE" ]] || return 0 + + while IFS=$'\t' read -r oracle avg n; do + [[ -n "$oracle" ]] && + printf 'average speedup vs %s wc: %s (%s cases)\n' \ + "$oracle" "$avg" "$n" + done <<< "$(awk -F'\t' ' + { sum[$1] += $2; n[$1]++ } + END { + if (n["coreutils"]) + printf "coreutils\t%.2fx\t%d\n", sum["coreutils"] / n["coreutils"], n["coreutils"]; + if (n["busybox"]) + printf "busybox\t%.2fx\t%d\n", sum["busybox"] / n["busybox"], n["busybox"]; + }' "$RATIOS_FILE")" +} diff --git a/benchmarks/stdin/piping/bench.sh b/benchmarks/stdin/piping/bench.sh index c7ce272..6b10d4f 100755 --- a/benchmarks/stdin/piping/bench.sh +++ b/benchmarks/stdin/piping/bench.sh @@ -2,9 +2,10 @@ # bench.sh — race fastwc against GNU coreutils wc on standard input, fed by # shell redirect (the same path every real pipeline uses). Counts must agree # with coreutils' stdin behavior, and fastwc must not be slower. -# Fails fast: the moment fastwc is slower than (or disagrees with) coreutils -# wc, a human readable report is written to FAILED-benchmark.txt and this -# script exits non-zero. +# Busybox wc is raced too, for shits and giggles. +# Fails fast: the moment fastwc is slower than (or disagrees with) an +# oracle wc, a human readable report is written to FAILED-benchmark.txt +# and this script exits non-zero. # # usage: ./bench.sh set -u @@ -15,26 +16,29 @@ source "$REPO_DIR/benchmarks/std.sh" checkfastwc checkwc -BENCH_NAME="coreutils" -printf 'benchmarking %s wc vs fastwc: stdin (%s interleaved runs each, minimum kept)\n' \ - "$BENCH_NAME" "$BENCH_REPS" -printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' +for oracle in $ORACLES; do + select_oracle "$oracle" || exit 1 -printf '%s\n' '--- stdin lines ---' -if ! run_stdin_cases lines -l 10000 100000 1000000 10000000; then - exit 1 -fi + printf 'benchmarking %s wc vs fastwc: stdin (%s interleaved runs each, minimum kept)\n' \ + "$BENCH_NAME" "$BENCH_REPS" + printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' -printf '%s\n' '--- stdin words ---' -if ! run_stdin_cases words -w 1000 100000 1000000; then - exit 1 -fi + printf '%s\n' '--- stdin lines ---' + if ! run_stdin_cases lines -l 10000 100000 1000000 10000000; then + exit 1 + fi -printf '%s\n' '--- stdin lines+words+bytes ---' -if ! run_stdin_cases lines -lwc 1000000; then - exit 1 -fi + printf '%s\n' '--- stdin words ---' + if ! run_stdin_cases words -w 1000 100000 1000000; then + exit 1 + fi -printf '\nall %s stdin benchmarks passed — fastwc was never slower than %s wc\n' \ - "$BENCH_NAME" "$BENCH_NAME" + printf '%s\n' '--- stdin lines+words+bytes ---' + if ! run_stdin_cases lines -lwc 1000000; then + exit 1 + fi + + printf '\nall %s stdin benchmarks passed — fastwc was never slower than %s wc\n' \ + "$BENCH_NAME" "$BENCH_NAME" +done diff --git a/benchmarks/test-all.sh b/benchmarks/test-all.sh index c6623b0..3690373 100755 --- a/benchmarks/test-all.sh +++ b/benchmarks/test-all.sh @@ -1,12 +1,22 @@ #!/usr/bin/env bash # test-all.sh — compile the benchmark helper tools first, then run every -# benchmark suite (words, lines incl. monsters, stdin). Exits non-zero if -# any of them fails. +# benchmark suite (words, lines incl. monsters, stdin) against every +# oracle found (coreutils, plus busybox for shits and giggles). Exits +# non-zero if any of them fails, and ends with the average speedup of +# fastwc against each oracle. # # usage: ./test-all.sh set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Share one ratios file across all suites so the final average covers +# every case; each suite still races through its own process. +RATIOS_FILE="$(mktemp /tmp/fastwc-ratios.XXXXXX)" +export RATIOS_FILE + +source "$SCRIPT_DIR/std.sh" # 1. build the helper tools before running any benchmark so createtxt() # can use the fast C generator instead of the slow shell fallback @@ -29,6 +39,10 @@ rc_stdin=$? if [[ $rc_words -ne 0 || $rc_lines -ne 0 || $rc_stdin -ne 0 ]]; then printf '\ntest-all: FAILED (words=%s, lines=%s, stdin=%s)\n' \ "$rc_words" "$rc_lines" "$rc_stdin" >&2 + print_averages + rm -f "$RATIOS_FILE" exit 1 fi printf '\ntest-all: all benchmarks passed\n' +print_averages +rm -f "$RATIOS_FILE"