bench: microsecond timing via tools/timeit, toybox oracle, words to 10M lines
This commit is contained in:
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# bench.sh — race fastwc against GNU coreutils wc on line counts, file input.
|
||||
# Busybox wc is raced too, for shits and giggles. Includes the monster
|
||||
# cases: 100M lines raced against coreutils (busybox through 1.1 GB is a
|
||||
# spectator sport, not a benchmark), and 1B lines timed solo (no
|
||||
# reference to beat — there isn't one).
|
||||
# Busybox and toybox wc are raced too, for shits and giggles. Includes
|
||||
# the monster cases: 100M lines raced against coreutils (busybox/toybox
|
||||
# through 1.1 GB is a spectator sport, not a benchmark), and 1B lines
|
||||
# timed solo (no reference to beat — there isn't one).
|
||||
# Fails fast: the moment fastwc is slower than (or disagrees with) an
|
||||
# oracle wc, a human readable report is written to FAILED-benchmark.txt
|
||||
# and this script exits non-zero.
|
||||
@@ -23,7 +23,7 @@ for oracle in $ORACLES; do
|
||||
|
||||
printf 'benchmarking %s wc vs fastwc: lines, file input (%s interleaved runs each, minimum kept)\n' \
|
||||
"$BENCH_NAME" "$BENCH_REPS"
|
||||
printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status'
|
||||
printf '%-28s %16s %24s %8s %s\n' 'test' 'wc' 'fastwc (µs)' 'ratio' 'status'
|
||||
|
||||
printf '%s\n' '--- lines ---'
|
||||
if ! run_cases lines -l 10000 100000 1000000 10000000; then
|
||||
@@ -34,8 +34,9 @@ for oracle in $ORACLES; do
|
||||
"$BENCH_NAME" "$BENCH_NAME"
|
||||
done
|
||||
|
||||
# The monsters stay coreutils-only (busybox through 1.1 GB is a spectator
|
||||
# sport, not a benchmark), so switch back from the last oracle raced.
|
||||
# The monsters stay coreutils-only (busybox/toybox through 1.1 GB is a
|
||||
# spectator sport, not a benchmark), so switch back from the last oracle
|
||||
# raced.
|
||||
select_oracle coreutils || exit 1
|
||||
printf '%s\n' '--- monster: 100M lines vs coreutils ---'
|
||||
if ! run_cases lines -l 100000000; then
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# bench.sh — race fastwc against GNU coreutils wc on word counts, file input.
|
||||
# Busybox wc is raced too, for shits and giggles.
|
||||
# Busybox and toybox wc are raced too, for shits and giggles.
|
||||
# Fails fast: the moment fastwc is slower than (or disagrees with) an
|
||||
# oracle wc, a human readable report is written to FAILED-benchmark.txt
|
||||
# and this script exits non-zero.
|
||||
@@ -20,10 +20,10 @@ for oracle in $ORACLES; do
|
||||
|
||||
printf 'benchmarking %s wc vs fastwc: words, file input (%s interleaved runs each, minimum kept)\n' \
|
||||
"$BENCH_NAME" "$BENCH_REPS"
|
||||
printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status'
|
||||
printf '%-28s %16s %24s %8s %s\n' 'test' 'wc' 'fastwc (µs)' 'ratio' 'status'
|
||||
|
||||
printf '%s\n' '--- words ---'
|
||||
if ! run_cases words -w 1 10 100 1000 10000 100000; then
|
||||
if ! run_cases words -w 1 10 100 1000 10000 100000 1000000 10000000; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
+104
-123
@@ -12,7 +12,7 @@
|
||||
# checkfastwc() verify a release build of fastwc exists
|
||||
# checkwc() locate the coreutils wc implementation
|
||||
# createtxt() create (or reuse) a text file with N such lines
|
||||
# time_ms() run a command once, print elapsed wall time in ms
|
||||
# timeit() run a command once, print elapsed wall time in µs
|
||||
# capture_count() print the first whitespace-separated field of output
|
||||
# run_case() race fastwc against the reference on a file argument
|
||||
# run_stdin_case() same, but feeding the file through standard input
|
||||
@@ -36,9 +36,14 @@ export LC_CTYPE=C
|
||||
FASTWC="$REPO_DIR/bin/release/fastwc"
|
||||
DATA_DIR="$SCRIPT_DIR/.data"
|
||||
GENFILE="$REPO_DIR/benchmarks/tools/genfile" # optional C helper, built by test-all.sh
|
||||
TIMEIT="$REPO_DIR/benchmarks/tools/timeit" # µs exec timer, built by test-all.sh
|
||||
|
||||
BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils
|
||||
BENCH_REPS="${BENCH_REPS:-3}" # interleaved runs per case; minimum is kept
|
||||
# A reference wc that finishes under this many µs was really measuring
|
||||
# startup, not throughput. Those cases are checked for correctness and
|
||||
# reported, but excluded from the averages (see finish_race).
|
||||
RACE_FLOOR_US="${RACE_FLOOR_US:-5000}"
|
||||
RESULT_ROWS="" # accumulated results table
|
||||
|
||||
# Per-case speedup ratios accumulate here so the run can end with the
|
||||
@@ -68,14 +73,16 @@ checkfastwc() {
|
||||
}
|
||||
|
||||
# wc_impl <cmd...> — identify a wc implementation from its --version
|
||||
# answer. coreutils and fastwc respond to --version; busybox does not
|
||||
# have the option and names itself in the error it prints instead.
|
||||
# answer. coreutils, fastwc and toybox respond to --version; busybox
|
||||
# does not have the option and names itself in the error it prints
|
||||
# instead.
|
||||
wc_impl() {
|
||||
local out
|
||||
out=$("$@" --version 2>&1)
|
||||
case "$out" in
|
||||
*'GNU coreutils'*) printf 'coreutils\n' ;;
|
||||
fastwc*) printf 'fastwc\n' ;;
|
||||
toybox*) printf 'toybox\n' ;;
|
||||
*BusyBox*) printf 'busybox\n' ;;
|
||||
*) printf 'unknown\n' ;;
|
||||
esac
|
||||
@@ -85,15 +92,17 @@ wc_impl() {
|
||||
# for every wc binary and identifies each by its --version answer. A wc
|
||||
# that answers as fastwc is a symlink somebody made to our own binary —
|
||||
# people do symlink wc to fastwc — and is skipped, because racing
|
||||
# ourselves proves nothing. Busybox usually exists only as the
|
||||
# multi-call binary, so that is probed too; it is back in the suite for
|
||||
# shits and giggles, not because it is a challenge.
|
||||
# Sets COREUTILS_CMD, BUSYBOX_CMD (empty if absent) and ORACLES, and
|
||||
# points WC_CMD at coreutils. Exits if no coreutils wc is found.
|
||||
# ourselves proves nothing. Busybox and toybox usually exist only as the
|
||||
# multi-call binaries, so those are probed too; they are back in the
|
||||
# suite for shits and giggles, not because they are a challenge.
|
||||
# Sets COREUTILS_CMD, BUSYBOX_CMD, TOYBOX_CMD (empty if absent) and
|
||||
# ORACLES, and points WC_CMD at coreutils. Exits if no coreutils wc is
|
||||
# found.
|
||||
checkwc() {
|
||||
local dir impl
|
||||
COREUTILS_CMD=()
|
||||
BUSYBOX_CMD=()
|
||||
TOYBOX_CMD=()
|
||||
ORACLES=''
|
||||
|
||||
printf 'locating wc implementations... '
|
||||
@@ -118,6 +127,12 @@ checkwc() {
|
||||
printf 'busybox; '
|
||||
fi
|
||||
;;
|
||||
toybox)
|
||||
if [[ ${#TOYBOX_CMD[@]} -eq 0 ]]; then
|
||||
TOYBOX_CMD=("$dir/wc")
|
||||
printf 'toybox; '
|
||||
fi
|
||||
;;
|
||||
*) ;;
|
||||
esac
|
||||
done
|
||||
@@ -129,6 +144,13 @@ checkwc() {
|
||||
printf 'busybox; '
|
||||
fi
|
||||
|
||||
# same for toybox
|
||||
if [[ ${#TOYBOX_CMD[@]} -eq 0 ]] && command -v toybox >/dev/null 2>&1 \
|
||||
&& [[ "$(wc_impl toybox wc)" == 'toybox' ]]; then
|
||||
TOYBOX_CMD=(toybox wc)
|
||||
printf 'toybox; '
|
||||
fi
|
||||
|
||||
if [[ ${#COREUTILS_CMD[@]} -eq 0 ]]; then
|
||||
printf 'none\n'
|
||||
printf 'configure: error: no coreutils wc found in PATH\n' >&2
|
||||
@@ -140,14 +162,16 @@ checkwc() {
|
||||
WC_CMD=("${COREUTILS_CMD[@]}")
|
||||
ORACLES='coreutils'
|
||||
[[ ${#BUSYBOX_CMD[@]} -gt 0 ]] && ORACLES="$ORACLES busybox"
|
||||
[[ ${#TOYBOX_CMD[@]} -gt 0 ]] && ORACLES="$ORACLES toybox"
|
||||
}
|
||||
|
||||
# select_oracle <coreutils|busybox> — point the racing functions at the
|
||||
# chosen oracle by setting BENCH_NAME and WC_CMD.
|
||||
# select_oracle <coreutils|busybox|toybox> — point the racing functions
|
||||
# at the chosen oracle by setting BENCH_NAME and WC_CMD.
|
||||
select_oracle() {
|
||||
case "$1" in
|
||||
coreutils) BENCH_NAME='coreutils'; WC_CMD=("${COREUTILS_CMD[@]}") ;;
|
||||
busybox) BENCH_NAME='busybox'; WC_CMD=("${BUSYBOX_CMD[@]}") ;;
|
||||
toybox) BENCH_NAME='toybox'; WC_CMD=("${TOYBOX_CMD[@]}") ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
return 0
|
||||
@@ -199,24 +223,17 @@ createtxt() {
|
||||
printf '%s\n' "$TEXT_FILE"
|
||||
}
|
||||
|
||||
# time_ms <cmd...> — run a command once and print elapsed wall time in ms.
|
||||
time_ms() {
|
||||
local s e
|
||||
s=$(date +%s%N)
|
||||
"$@" >/dev/null 2>&1
|
||||
e=$(date +%s%N)
|
||||
printf '%s\n' "$(( (e - s) / 1000000 ))"
|
||||
}
|
||||
|
||||
# time_us <cmd...> — run a command once and print elapsed wall time in µs.
|
||||
# More precise than time_ms: sub-millisecond runs come out as e.g. 812,
|
||||
# not 0. The benchmark keeps the µs reading for fastwc and derives the ms.
|
||||
time_us() {
|
||||
local s e
|
||||
s=$(date +%s%N)
|
||||
"$@" >/dev/null 2>&1
|
||||
e=$(date +%s%N)
|
||||
printf '%s\n' "$(( (e - s) / 1000 ))"
|
||||
# timeit <cmd...> — run a command once and print elapsed wall time in µs
|
||||
# on stdout, with the command's own output discarded. The clock starts in
|
||||
# main(), after timeit's own loader has run, so every binary pays the same
|
||||
# bookkeeping and the reading is the child's fork + exec + run + exit.
|
||||
timeit() {
|
||||
[[ -x "$TIMEIT" ]] || {
|
||||
printf 'error: %s not built — run test-all.sh (or "make bench") first\n' \
|
||||
"$TIMEIT" >&2
|
||||
exit 1
|
||||
}
|
||||
"$TIMEIT" "$@"
|
||||
}
|
||||
|
||||
# capture_count <cmd...> — print the first whitespace-separated field of a
|
||||
@@ -225,10 +242,10 @@ capture_count() {
|
||||
"$@" 2>/dev/null | awk 'NR == 1 { print $1 }'
|
||||
}
|
||||
|
||||
# write_failed_report <label> <reason> <wc_ms> <fast_ms> — write the
|
||||
# write_failed_report <label> <reason> <wc_us> <fast_us> — write the
|
||||
# human readable failure report to FAILED-benchmark.txt next to the suite.
|
||||
write_failed_report() {
|
||||
local label="$1" reason="$2" wc_ms="$3" fast_ms="$4"
|
||||
local label="$1" reason="$2" wc_us="$3" fast_us="$4"
|
||||
local report="$SCRIPT_DIR/FAILED-benchmark.txt"
|
||||
|
||||
{
|
||||
@@ -239,7 +256,7 @@ write_failed_report() {
|
||||
printf 'failure : %s\n' "$reason"
|
||||
printf '\nresults\n'
|
||||
printf '%s\n' '-------'
|
||||
printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status'
|
||||
printf '%-28s %16s %24s %8s %s\n' 'test' 'wc' 'fastwc (µs)' 'ratio' 'status'
|
||||
printf '%s' "$RESULT_ROWS"
|
||||
printf '\nfastwc must never be slower than %s wc — benchmark aborted.\n' "$BENCH_NAME"
|
||||
} > "$report"
|
||||
@@ -248,6 +265,45 @@ write_failed_report() {
|
||||
printf 'full results written to %s\n' "$report" >&2
|
||||
}
|
||||
|
||||
# finish_race <label> <wc_count> <fast_count> <wc_us> <fast_us> — turn the
|
||||
# measured counts and times into a verdict. A count mismatch fails
|
||||
# outright. A reference that finished under RACE_FLOOR_US was really
|
||||
# measuring startup, not throughput: the case is reported as PASS
|
||||
# (startup-bound) and left out of the averages. Raced cases pass when
|
||||
# fastwc is not slower than the reference beyond a small dead-heat slack
|
||||
# (200µs + 2%) that absorbs scheduler jitter on genuinely even races.
|
||||
finish_race() {
|
||||
local label="$1" wc_count="$2" fast_count="$3" wc_us="$4" fast_us="$5"
|
||||
local ratio='-' verdict='PASS' reason='' row slack
|
||||
|
||||
if [[ "$fast_count" != "$wc_count" ]]; then
|
||||
verdict='FAIL'
|
||||
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
|
||||
elif (( wc_us < RACE_FLOOR_US )); then
|
||||
verdict='PASS (startup-bound)'
|
||||
else
|
||||
ratio=$(awk -v f="$fast_us" -v w="$wc_us" 'BEGIN { printf "%.2fx", w / f }')
|
||||
# feed the end-of-run average speedup (raced rows only)
|
||||
printf '%s\t%s\n' "$BENCH_NAME" "${ratio%x}" >> "$RATIOS_FILE"
|
||||
slack=$((200 + wc_us / 50))
|
||||
if (( fast_us > wc_us + slack )); then
|
||||
verdict='FAIL'
|
||||
reason="fastwc was slower (fastwc: ${fast_us}µs vs ${BENCH_NAME} wc: ${wc_us}µs)"
|
||||
fi
|
||||
fi
|
||||
|
||||
row=$(printf '%-28s %16s %24s %8s %s\n' \
|
||||
"$label" "wc: ${wc_us}µs" "fastwc: ${fast_us}µs" "$ratio" "$verdict")
|
||||
RESULT_ROWS+="${row}"$'\n'
|
||||
printf '%s\n' "$row"
|
||||
|
||||
if [[ "$verdict" == 'FAIL' ]]; then
|
||||
write_failed_report "$label" "$reason" "$wc_us" "$fast_us"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# run_case <words|lines> <n-lines> <-w|-l> — race the reference wc against
|
||||
# fastwc on a file argument. Fails the benchmark the moment fastwc is
|
||||
# slower or reports a different count.
|
||||
@@ -255,7 +311,7 @@ run_case() {
|
||||
local mode="$1" lines="$2" flag="$3"
|
||||
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
|
||||
local label="${mode} (${lines} ${noun})"
|
||||
local file wc_count fast_count wc_ms fast_us fast_ms ratio verdict reason row
|
||||
local file wc_count fast_count wc_us fast_us
|
||||
local i d
|
||||
|
||||
file=$(createtxt "$lines") || return 1
|
||||
@@ -264,58 +320,19 @@ run_case() {
|
||||
wc_count=$(capture_count "${WC_CMD[@]}" "$flag" "$file")
|
||||
fast_count=$(capture_count "$FASTWC" "$flag" "$file")
|
||||
|
||||
# speed: interleaved timing so both commands see identical cache warmth;
|
||||
# keep the minimum of $BENCH_REPS runs each to reduce noise; fastwc is
|
||||
# timed in µs so sub-millisecond wins are visible in the report
|
||||
wc_ms=''
|
||||
# speed: interleaved timing so both commands see identical cache
|
||||
# warmth; keep the minimum of $BENCH_REPS runs each
|
||||
wc_us=''
|
||||
fast_us=''
|
||||
for ((i = 0; i < BENCH_REPS; i++)); do
|
||||
d=$(time_ms "${WC_CMD[@]}" "$flag" "$file")
|
||||
[[ -z "$wc_ms" || "$d" -lt "$wc_ms" ]] && wc_ms="$d"
|
||||
d=$(timeit "${WC_CMD[@]}" "$flag" "$file")
|
||||
[[ -z "$wc_us" || "$d" -lt "$wc_us" ]] && wc_us="$d"
|
||||
|
||||
d=$(time_us "$FASTWC" "$flag" "$file")
|
||||
d=$(timeit "$FASTWC" "$flag" "$file")
|
||||
[[ -z "$fast_us" || "$d" -lt "$fast_us" ]] && fast_us="$d"
|
||||
done
|
||||
fast_ms=$(( fast_us / 1000 ))
|
||||
|
||||
# ratio: how many times faster fastwc is than the reference (wc / fastwc)
|
||||
if [[ -n "$wc_ms" && "$wc_ms" -gt 0 ]]; then
|
||||
if [[ "$fast_ms" -gt 0 ]]; then
|
||||
ratio=$(awk -v f="$fast_ms" -v w="$wc_ms" 'BEGIN { printf "%.2fx", w / f }')
|
||||
else
|
||||
ratio='infx'
|
||||
fi
|
||||
else
|
||||
ratio='-'
|
||||
fi
|
||||
# feed the end-of-run average speedup (numeric ratios only)
|
||||
if [[ "$ratio" == *x && "$ratio" != 'infx' ]]; then
|
||||
printf '%s\t%s\n' "$BENCH_NAME" "${ratio%x}" >> "$RATIOS_FILE"
|
||||
fi
|
||||
|
||||
verdict='PASS'
|
||||
reason=''
|
||||
if [[ "$fast_count" != "$wc_count" ]]; then
|
||||
verdict='FAIL'
|
||||
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
|
||||
elif (( wc_ms > 0 && fast_ms > wc_ms )); then
|
||||
# a 0ms reference is below the benchmark's resolution: sub-millisecond
|
||||
# runs (startup noise, mostly) cannot prove fastwc slower
|
||||
verdict='FAIL'
|
||||
reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)"
|
||||
fi
|
||||
|
||||
row=$(printf '%-28s %10s %22s %8s %s\n' \
|
||||
"$label" "wc: ${wc_ms}ms" "fastwc: ${fast_ms}ms (${fast_us}µs)" \
|
||||
"$ratio" "$verdict")
|
||||
RESULT_ROWS+="${row}"$'\n'
|
||||
printf '%s\n' "$row"
|
||||
|
||||
if [[ "$verdict" == 'FAIL' ]]; then
|
||||
write_failed_report "$label" "$reason" "$wc_ms" "$fast_ms"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
finish_race "$label" "$wc_count" "$fast_count" "$wc_us" "$fast_us"
|
||||
}
|
||||
|
||||
# run_stdin_case <words|lines> <n-lines> <-w|-l|...> — same race, but the
|
||||
@@ -325,7 +342,7 @@ run_stdin_case() {
|
||||
local mode="$1" lines="$2" flag="$3"
|
||||
local noun='lines'; [[ "$lines" -eq 1 ]] && noun='line'
|
||||
local label="stdin ${mode} (${lines} ${noun})"
|
||||
local file wc_count fast_count wc_ms fast_us fast_ms ratio verdict reason row
|
||||
local file wc_count fast_count wc_us fast_us
|
||||
local i d
|
||||
|
||||
file=$(createtxt "$lines") || return 1
|
||||
@@ -333,55 +350,17 @@ run_stdin_case() {
|
||||
wc_count=$(capture_count "${WC_CMD[@]}" "$flag" < "$file")
|
||||
fast_count=$(capture_count "$FASTWC" "$flag" < "$file")
|
||||
|
||||
wc_ms=''
|
||||
wc_us=''
|
||||
fast_us=''
|
||||
for ((i = 0; i < BENCH_REPS; i++)); do
|
||||
d=$(time_ms "${WC_CMD[@]}" "$flag" < "$file")
|
||||
[[ -z "$wc_ms" || "$d" -lt "$wc_ms" ]] && wc_ms="$d"
|
||||
d=$(timeit "${WC_CMD[@]}" "$flag" < "$file")
|
||||
[[ -z "$wc_us" || "$d" -lt "$wc_us" ]] && wc_us="$d"
|
||||
|
||||
d=$(time_us "$FASTWC" "$flag" < "$file")
|
||||
d=$(timeit "$FASTWC" "$flag" < "$file")
|
||||
[[ -z "$fast_us" || "$d" -lt "$fast_us" ]] && fast_us="$d"
|
||||
done
|
||||
fast_ms=$(( fast_us / 1000 ))
|
||||
|
||||
# ratio: how many times faster fastwc is than the reference (wc / fastwc)
|
||||
if [[ -n "$wc_ms" && "$wc_ms" -gt 0 ]]; then
|
||||
if [[ "$fast_ms" -gt 0 ]]; then
|
||||
ratio=$(awk -v f="$fast_ms" -v w="$wc_ms" 'BEGIN { printf "%.2fx", w / f }')
|
||||
else
|
||||
ratio='infx'
|
||||
fi
|
||||
else
|
||||
ratio='-'
|
||||
fi
|
||||
# feed the end-of-run average speedup (numeric ratios only)
|
||||
if [[ "$ratio" == *x && "$ratio" != 'infx' ]]; then
|
||||
printf '%s\t%s\n' "$BENCH_NAME" "${ratio%x}" >> "$RATIOS_FILE"
|
||||
fi
|
||||
|
||||
verdict='PASS'
|
||||
reason=''
|
||||
if [[ "$fast_count" != "$wc_count" ]]; then
|
||||
verdict='FAIL'
|
||||
reason="output mismatch (fastwc: ${fast_count}, ${BENCH_NAME} wc: ${wc_count})"
|
||||
elif (( wc_ms > 0 && fast_ms > wc_ms )); then
|
||||
# a 0ms reference is below the benchmark's resolution: sub-millisecond
|
||||
# runs (startup noise, mostly) cannot prove fastwc slower
|
||||
verdict='FAIL'
|
||||
reason="fastwc was slower (fastwc: ${fast_ms}ms vs ${BENCH_NAME} wc: ${wc_ms}ms)"
|
||||
fi
|
||||
|
||||
row=$(printf '%-28s %10s %22s %8s %s\n' \
|
||||
"$label" "wc: ${wc_ms}ms" "fastwc: ${fast_ms}ms (${fast_us}µs)" \
|
||||
"$ratio" "$verdict")
|
||||
RESULT_ROWS+="${row}"$'\n'
|
||||
printf '%s\n' "$row"
|
||||
|
||||
if [[ "$verdict" == 'FAIL' ]]; then
|
||||
write_failed_report "$label" "$reason" "$wc_ms" "$fast_ms"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
finish_race "$label" "$wc_count" "$fast_count" "$wc_us" "$fast_us"
|
||||
}
|
||||
|
||||
# run_solo_case <n-lines> <-l|...> — time fastwc alone on <n-lines> of data,
|
||||
@@ -399,7 +378,7 @@ run_solo_case() {
|
||||
}
|
||||
|
||||
for ((i = 0; i < BENCH_REPS; i++)); do
|
||||
d=$(time_us "$FASTWC" "$flag" "$file")
|
||||
d=$(timeit "$FASTWC" "$flag" "$file")
|
||||
[[ -z "$best_us" || "$d" -lt "$best_us" ]] && best_us="$d"
|
||||
done
|
||||
best_ms=$(( best_us / 1000 ))
|
||||
@@ -455,5 +434,7 @@ print_averages() {
|
||||
printf "coreutils\t%.2fx\t%d\n", sum["coreutils"] / n["coreutils"], n["coreutils"];
|
||||
if (n["busybox"])
|
||||
printf "busybox\t%.2fx\t%d\n", sum["busybox"] / n["busybox"], n["busybox"];
|
||||
if (n["toybox"])
|
||||
printf "toybox\t%.2fx\t%d\n", sum["toybox"] / n["toybox"], n["toybox"];
|
||||
}' "$RATIOS_FILE")"
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ for oracle in $ORACLES; do
|
||||
|
||||
printf 'benchmarking %s wc vs fastwc: stdin (%s interleaved runs each, minimum kept)\n' \
|
||||
"$BENCH_NAME" "$BENCH_REPS"
|
||||
printf '%-28s %10s %22s %8s %s\n' 'test' 'wc' 'fastwc (ms µs)' 'ratio' 'status'
|
||||
printf '%-28s %16s %24s %8s %s\n' 'test' 'wc' 'fastwc (µs)' 'ratio' 'status'
|
||||
|
||||
printf '%s\n' '--- stdin lines ---'
|
||||
if ! run_stdin_cases lines -l 10000 100000 1000000 10000000; then
|
||||
|
||||
+10
-4
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-all.sh — compile the benchmark helper tools first, then run every
|
||||
# benchmark suite (words, lines incl. monsters, stdin) against every
|
||||
# oracle found (coreutils, plus busybox for shits and giggles). Exits
|
||||
# non-zero if any of them fails, and ends with the average speedup of
|
||||
# fastwc against each oracle.
|
||||
# oracle found (coreutils, plus busybox and toybox for shits and
|
||||
# giggles). Exits non-zero if any of them fails, and ends with the
|
||||
# average speedup of fastwc against each oracle.
|
||||
#
|
||||
# usage: ./test-all.sh
|
||||
set -u
|
||||
@@ -19,13 +19,19 @@ export RATIOS_FILE
|
||||
source "$SCRIPT_DIR/std.sh"
|
||||
|
||||
# 1. build the helper tools before running any benchmark so createtxt()
|
||||
# can use the fast C generator instead of the slow shell fallback
|
||||
# can use the fast C generator instead of the slow shell fallback and
|
||||
# the races are timed at µs resolution
|
||||
mkdir -p "$SCRIPT_DIR/tools"
|
||||
if ! cc -O2 -Wall -o "$SCRIPT_DIR/tools/genfile" "$SCRIPT_DIR/tools/genfile.c"; then
|
||||
printf 'test-all: error: failed to compile %s\n' "$SCRIPT_DIR/tools/genfile.c" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'built %s\n' "$SCRIPT_DIR/tools/genfile"
|
||||
if ! cc -O2 -Wall -o "$SCRIPT_DIR/tools/timeit" "$SCRIPT_DIR/tools/timeit.c"; then
|
||||
printf 'test-all: error: failed to compile %s\n' "$SCRIPT_DIR/tools/timeit.c" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'built %s\n' "$SCRIPT_DIR/tools/timeit"
|
||||
|
||||
# 2. run each suite (all run regardless, so every result is reported)
|
||||
"$SCRIPT_DIR/files/words/bench.sh"
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
/* timeit — run a command once and print its elapsed wall time in µs.
|
||||
*
|
||||
* The benchmark harness races fastwc against several wc implementations.
|
||||
* Timing both sides by wrapping the command in `date +%s%N` forks added
|
||||
* over a millisecond of noise per sample — more than the whole run on a
|
||||
* small case — so every sub-millisecond race was decided by fork jitter,
|
||||
* not by speed. This helper measures a plain fork + exec + wait with
|
||||
* clock_gettime and prints the elapsed microseconds on its own stdout.
|
||||
* The timed command's output is discarded, exactly as the old date
|
||||
* wrapper did, so capture_count (which runs the command directly) is the
|
||||
* only path that sees real output.
|
||||
*
|
||||
* usage: timeit <cmd> [arg...]
|
||||
*/
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <sys/wait.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
struct timespec t0, t1;
|
||||
pid_t pid;
|
||||
int nullfd;
|
||||
|
||||
if (argc < 2)
|
||||
return 2;
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC, &t0);
|
||||
|
||||
pid = fork();
|
||||
if (pid < 0)
|
||||
return 2;
|
||||
if (pid == 0)
|
||||
{
|
||||
/* child: run the timed command with its output thrown away */
|
||||
nullfd = open("/dev/null", O_WRONLY);
|
||||
if (nullfd >= 0)
|
||||
{
|
||||
dup2(nullfd, STDOUT_FILENO);
|
||||
dup2(nullfd, STDERR_FILENO);
|
||||
close(nullfd);
|
||||
}
|
||||
execvp(argv[1], &argv[1]);
|
||||
_exit(127);
|
||||
}
|
||||
|
||||
if (waitpid(pid, NULL, 0) < 0)
|
||||
return 2;
|
||||
clock_gettime(CLOCK_MONOTONIC, &t1);
|
||||
|
||||
printf("%lld\n",
|
||||
((long long)(t1.tv_sec - t0.tv_sec) * 1000000000LL
|
||||
+ (t1.tv_nsec - t0.tv_nsec)) / 1000);
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user