diff --git a/.gitignore b/.gitignore index 5179f75..12cd1f0 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ dkms.conf # ---> fastwc benchmarks benchmarks/tools/genfile +benchmarks/tools/timeit benchmarks/.data/ benchmarks/files/*/.data/ benchmarks/stdin/piping/.data/ diff --git a/benchmarks/files/lines/bench.sh b/benchmarks/files/lines/bench.sh index 33fe1fd..3561154 100755 --- a/benchmarks/files/lines/bench.sh +++ b/benchmarks/files/lines/bench.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # bench.sh — race fastwc against GNU coreutils wc on line counts, file input. -# Busybox wc is raced too, for shits and giggles. Includes the monster -# cases: 100M lines raced against coreutils (busybox through 1.1 GB is a -# spectator sport, not a benchmark), and 1B lines timed solo (no -# reference to beat — there isn't one). +# Busybox and toybox wc are raced too, for shits and giggles. Includes +# the monster cases: 100M lines raced against coreutils (busybox/toybox +# through 1.1 GB is a spectator sport, not a benchmark), and 1B lines +# timed solo (no reference to beat — there isn't one). # Fails fast: the moment fastwc is slower than (or disagrees with) an # oracle wc, a human readable report is written to FAILED-benchmark.txt # and this script exits non-zero. @@ -23,7 +23,7 @@ for oracle in $ORACLES; do printf 'benchmarking %s wc vs fastwc: lines, file input (%s interleaved runs each, minimum kept)\n' \ "$BENCH_NAME" "$BENCH_REPS" - printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' + printf '%-28s %16s %24s %8s %s\n' 'test' 'wc' 'fastwc (µs)' 'ratio' 'status' printf '%s\n' '--- lines ---' if ! run_cases lines -l 10000 100000 1000000 10000000; then @@ -34,8 +34,9 @@ for oracle in $ORACLES; do "$BENCH_NAME" "$BENCH_NAME" done -# The monsters stay coreutils-only (busybox through 1.1 GB is a spectator -# sport, not a benchmark), so switch back from the last oracle raced. +# The monsters stay coreutils-only (busybox/toybox through 1.1 GB is a +# spectator sport, not a benchmark), so switch back from the last oracle +# raced. select_oracle coreutils || exit 1 printf '%s\n' '--- monster: 100M lines vs coreutils ---' if ! run_cases lines -l 100000000; then diff --git a/benchmarks/files/words/bench.sh b/benchmarks/files/words/bench.sh index 0b30649..dc38d44 100755 --- a/benchmarks/files/words/bench.sh +++ b/benchmarks/files/words/bench.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # bench.sh — race fastwc against GNU coreutils wc on word counts, file input. -# Busybox wc is raced too, for shits and giggles. +# Busybox and toybox wc are raced too, for shits and giggles. # Fails fast: the moment fastwc is slower than (or disagrees with) an # oracle wc, a human readable report is written to FAILED-benchmark.txt # and this script exits non-zero. @@ -20,10 +20,10 @@ for oracle in $ORACLES; do printf 'benchmarking %s wc vs fastwc: words, file input (%s interleaved runs each, minimum kept)\n' \ "$BENCH_NAME" "$BENCH_REPS" - printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status' + printf '%-28s %16s %24s %8s %s\n' 'test' 'wc' 'fastwc (µs)' 'ratio' 'status' printf '%s\n' '--- words ---' - if ! run_cases words -w 1 10 100 1000 10000 100000; then + if ! run_cases words -w 1 10 100 1000 10000 100000 1000000 10000000; then exit 1 fi diff --git a/benchmarks/std.sh b/benchmarks/std.sh index 1fa4d96..8a1d016 100755 --- a/benchmarks/std.sh +++ b/benchmarks/std.sh @@ -12,7 +12,7 @@ # checkfastwc() verify a release build of fastwc exists # checkwc() locate the coreutils wc implementation # createtxt() create (or reuse) a text file with N such lines -# time_ms() run a command once, print elapsed wall time in ms +# timeit() run a command once, print elapsed wall time in µs # capture_count() print the first whitespace-separated field of output # run_case() race fastwc against the reference on a file argument # run_stdin_case() same, but feeding the file through standard input @@ -36,9 +36,14 @@ export LC_CTYPE=C FASTWC="$REPO_DIR/bin/release/fastwc" DATA_DIR="$SCRIPT_DIR/.data" GENFILE="$REPO_DIR/benchmarks/tools/genfile" # optional C helper, built by test-all.sh +TIMEIT="$REPO_DIR/benchmarks/tools/timeit" # µs exec timer, built by test-all.sh BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils BENCH_REPS="${BENCH_REPS:-3}" # interleaved runs per case; minimum is kept +# A reference wc that finishes under this many µs was really measuring +# startup, not throughput. Those cases are checked for correctness and +# reported, but excluded from the averages (see finish_race). +RACE_FLOOR_US="${RACE_FLOOR_US:-5000}" RESULT_ROWS="" # accumulated results table # Per-case speedup ratios accumulate here so the run can end with the @@ -68,14 +73,16 @@ checkfastwc() { } # wc_impl — identify a wc implementation from its --version -# answer. coreutils and fastwc respond to --version; busybox does not -# have the option and names itself in the error it prints instead. +# answer. coreutils, fastwc and toybox respond to --version; busybox +# does not have the option and names itself in the error it prints +# instead. wc_impl() { local out out=$("$@" --version 2>&1) case "$out" in *'GNU coreutils'*) printf 'coreutils\n' ;; fastwc*) printf 'fastwc\n' ;; + toybox*) printf 'toybox\n' ;; *BusyBox*) printf 'busybox\n' ;; *) printf 'unknown\n' ;; esac @@ -85,15 +92,17 @@ wc_impl() { # for every wc binary and identifies each by its --version answer. A wc # that answers as fastwc is a symlink somebody made to our own binary — # people do symlink wc to fastwc — and is skipped, because racing -# ourselves proves nothing. Busybox usually exists only as the -# multi-call binary, so that is probed too; it is back in the suite for -# shits and giggles, not because it is a challenge. -# Sets COREUTILS_CMD, BUSYBOX_CMD (empty if absent) and ORACLES, and -# points WC_CMD at coreutils. Exits if no coreutils wc is found. +# ourselves proves nothing. Busybox and toybox usually exist only as the +# multi-call binaries, so those are probed too; they are back in the +# suite for shits and giggles, not because they are a challenge. +# Sets COREUTILS_CMD, BUSYBOX_CMD, TOYBOX_CMD (empty if absent) and +# ORACLES, and points WC_CMD at coreutils. Exits if no coreutils wc is +# found. checkwc() { local dir impl COREUTILS_CMD=() BUSYBOX_CMD=() + TOYBOX_CMD=() ORACLES='' printf 'locating wc implementations... ' @@ -118,6 +127,12 @@ checkwc() { printf 'busybox; ' fi ;; + toybox) + if [[ ${#TOYBOX_CMD[@]} -eq 0 ]]; then + TOYBOX_CMD=("$dir/wc") + printf 'toybox; ' + fi + ;; *) ;; esac done @@ -129,6 +144,13 @@ checkwc() { printf 'busybox; ' fi + # same for toybox + if [[ ${#TOYBOX_CMD[@]} -eq 0 ]] && command -v toybox >/dev/null 2>&1 \ + && [[ "$(wc_impl toybox wc)" == 'toybox' ]]; then + TOYBOX_CMD=(toybox wc) + printf 'toybox; ' + fi + if [[ ${#COREUTILS_CMD[@]} -eq 0 ]]; then printf 'none\n' printf 'configure: error: no coreutils wc found in PATH\n' >&2 @@ -140,14 +162,16 @@ checkwc() { WC_CMD=("${COREUTILS_CMD[@]}") ORACLES='coreutils' [[ ${#BUSYBOX_CMD[@]} -gt 0 ]] && ORACLES="$ORACLES busybox" + [[ ${#TOYBOX_CMD[@]} -gt 0 ]] && ORACLES="$ORACLES toybox" } -# select_oracle — point the racing functions at the -# chosen oracle by setting BENCH_NAME and WC_CMD. +# select_oracle — point the racing functions +# at the chosen oracle by setting BENCH_NAME and WC_CMD. select_oracle() { case "$1" in coreutils) BENCH_NAME='coreutils'; WC_CMD=("${COREUTILS_CMD[@]}") ;; busybox) BENCH_NAME='busybox'; WC_CMD=("${BUSYBOX_CMD[@]}") ;; + toybox) BENCH_NAME='toybox'; WC_CMD=("${TOYBOX_CMD[@]}") ;; *) return 1 ;; esac return 0 @@ -199,24 +223,17 @@ createtxt() { printf '%s\n' "$TEXT_FILE" } -# time_ms — run a command once and print elapsed wall time in ms. -time_ms() { - local s e - s=$(date +%s%N) - "$@" >/dev/null 2>&1 - e=$(date +%s%N) - printf '%s\n' "$(( (e - s) / 1000000 ))" -} - -# time_us — run a command once and print elapsed wall time in µs. -# More precise than time_ms: sub-millisecond runs come out as e.g. 812, -# not 0. The benchmark keeps the µs reading for fastwc and derives the ms. -time_us() { - local s e - s=$(date +%s%N) - "$@" >/dev/null 2>&1 - e=$(date +%s%N) - printf '%s\n' "$(( (e - s) / 1000 ))" +# timeit — run a command once and print elapsed wall time in µs +# on stdout, with the command's own output discarded. The clock starts in +# main(), after timeit's own loader has run, so every binary pays the same +# bookkeeping and the reading is the child's fork + exec + run + exit. +timeit() { + [[ -x "$TIMEIT" ]] || { + printf 'error: %s not built — run test-all.sh (or "make bench") first\n' \ + "$TIMEIT" >&2 + exit 1 + } + "$TIMEIT" "$@" } # capture_count — print the first whitespace-separated field of a @@ -225,10 +242,10 @@ capture_count() { "$@" 2>/dev/null | awk 'NR == 1 { print $1 }' } -# write_failed_report