From c4d4f0b71335ebd3e56898c614c5935dabe2e9fa Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Sat, 29 Aug 2026 13:48:15 -0400 Subject: [PATCH] first: setup setting up repo. --- .gitignore | 23 +++ CONTRIBUTING.md | 139 +++++++++++++ Makefile.am | 18 ++ README.md | 85 +++++++- STYLEGUIDE.md | 92 +++++++++ autogen.sh | 6 + benchmarks/bench-busybox.sh | 18 ++ benchmarks/bench-coreutils.sh | 18 ++ benchmarks/std.sh | 257 ++++++++++++++++++++++++ benchmarks/test-all.sh | 30 +++ benchmarks/tools/genfile.c | 70 +++++++ configure.ac | 10 + src/.dirstamp | 0 src/main.c | 358 ++++++++++++++++++++++++++++++++++ 14 files changed, 1123 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md create mode 100644 Makefile.am create mode 100644 STYLEGUIDE.md create mode 100755 autogen.sh create mode 100755 benchmarks/bench-busybox.sh create mode 100755 benchmarks/bench-coreutils.sh create mode 100755 benchmarks/std.sh create mode 100755 benchmarks/test-all.sh create mode 100644 benchmarks/tools/genfile.c create mode 100644 configure.ac create mode 100644 src/.dirstamp create mode 100644 src/main.c diff --git a/.gitignore b/.gitignore index cd531cf..9566867 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,26 @@ Module.symvers Mkfile.old dkms.conf +# ---> fastwc benchmarks +benchmarks/tools/genfile +benchmarks/.data/ +benchmarks/FAILED-benchmark.txt + +# ---> fastwc build artifacts +bin/ +fastwc + +# ---> autotools generated +Makefile +Makefile.in +aclocal.m4 +autom4te.cache/ +compile +config.log +config.status +configure +depcomp +.deps/ +install-sh +missing + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8d95aa5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,139 @@ +# Contributing to fastwc + +We're building the word counter GNU wc is still trying to be after forty +years. If that sounds like your kind of project, keep reading. + +## The rules + +These aren't guidelines. They're the deal. + +### 1. The benchmark is the contract + +`./benchmarks/bench-coreutils.sh` must pass. Not "mostly pass." Not +"pass on your machine." Pass. + +The moment fastwc is slower than GNU wc — or disagrees with it on a +single count — your change does not ship. We did not spend this much +effort being faster than a forty-year-old dependency museum just so you +could add a `strlen()` in the hot loop. + +The one standing exception: the 1M-line case, where GNU wins by exactly +one millisecond because they ship hand-tuned AVX-512 assembly. Closing +that gap is the project's open goal, not your excuse to be slower +anywhere else. + +### 2. C99, or don't bother + +We compile with `-Wall -Wextra` and zero warnings. Zero. If the compiler +is complaining, the compiler is right and you are wrong. We target C99 +with POSIX extensions, not whatever your compiler vendor's imagination +happens to ship this quarter. + +### 3. No new dependencies. Ever. + +The C standard library is the only thing we trust. No gettext. No gnulib. +No `libutf8pro` with its own translation team. GNU wc needs a build +system the size of a small country to count newlines. We need `cc`. + +If your feature needs a library, the feature is too big. Make the +feature smaller. + +### 4. The hot path is sacred + +`count_stream()`, `count_newlines()`, and `count_words()` in +`src/main.c` are the entire point of this project. You may touch them +only if `./benchmarks/bench-coreutils.sh` still passes afterward. +Rules for the counting loops: + +- fixed stride, no per-token function calls +- no allocation, no locking, no syscalls — the read is the only syscall +- branchless where it costs nothing; lookup tables are the foundation, + not a hack +- if you make it "clearer" but 2x slower, it's not clearer, it's worse + +### 5. Never suppress the truth + +No casts to silence warnings. No `-Wno-*` flags to make the noise go +away. No `void*` where a real type exists. If the type system is +fighting you, you're fighting the design. Fix the design. + +### 6. The codebase is one file. Keep it that way. + +`src/main.c` is one file and it will stay one file. GNU wc needs +forty-two translation units and a build system with its own mailing +list. We have one file and we can read all of it in one sitting. If your +feature needs a new file, your feature is over-engineered. + +### 7. Tests are the benchmarks + +There is no separate test suite, because the benchmark *is* the test +suite — it checks correctness against GNU wc (the only oracle around, +and it's wrong often enough to keep us humble) and speed in the same +breath. Add a flag? It gets benchmarked. Change the counting? It gets +benchmarked. The fail-fast scripts in `benchmarks/` are your tests, and +all of them must pass before you open a PR. + +### 8. No AI slop + +If it looks like ChatGPT wrote it, it gets rejected. We can tell. Write +code like a human who's been doing this for a decade and is tired of +GNU's excuses. + +## How to contribute + +### Pick something + +Good first issues: + +- Closing the 1M-line AVX-512 gap (the only loss on the board) +- Benchmark case coverage for `-m` and multiple files +- `bench-busybox.sh` CI on a machine that actually has busybox + +Ambitious issues: + +- A word counter that doesn't just tie GNU — it embarrasses it +- Bigger SWAR chunks, wider strides, less patience + +### Send a PR + +1. Fork the repo +2. Create a branch: `feat/my-thing` or `fix/my-bug` +3. Write code that follows the rules above +4. Run `./benchmarks/test-all.sh` — everything must pass +5. Open a PR against `main` + +### PR requirements + +- Build must pass with zero warnings: `make && make release` +- Benchmarks must pass: `./benchmarks/test-all.sh` +- Follow [STYLEGUIDE.md](STYLEGUIDE.md). The style is the law. +- No commented-out code. No dead code. No TODO without a date. +- Commit messages in imperative: `Count newlines in 8-byte strides` not + `Added newline counting optimization` + +## What we won't merge + +- **Anything slower than GNU wc.** We didn't come this far to tie. +- **Dependencies.** If it needs a library, it needs to not need a library. +- **GNU-style bloat.** Long options with help strings in fourteen + languages. Translation infrastructure. "Enterprise" anything. This is + a word counter, not a CRM. +- **Abstract nonsense.** Three layers of indirection to add a feature + means the feature is too complicated. GNU wc is what happens when you + let abstractions win. +- **AI slop.** See rule 8. + +## Communication + +We don't have a Discord. We don't have a forum. We don't have a mailing +list with a code of conduct and a weekly digest. Open an issue. Write a +clear title, a reproduction case, and what you expected. We'll respond +when we respond. + +If you want to propose a major feature, open an issue first. Surprise +PRs that rewrite the counting loops get closed without review — the +benchmark is the referee, and it doesn't know you. + +--- + +fastwc is 0.1.0. Everything is subject to change except the rules above. diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..4abe191 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,18 @@ +AM_CFLAGS = -Wall -Wextra -O2 + +bin_PROGRAMS = fastwc +fastwc_SOURCES = src/main.c + +# Release build consumed by benchmarks/ (expects bin/release/fastwc). +release: all + $(MKDIR_P) bin/release + cp -f fastwc bin/release/fastwc + +# Convenience: build the release binary, then run every benchmark suite. +bench: release + ./benchmarks/test-all.sh + +clean-local: + rm -rf bin + +.PHONY: release bench diff --git a/README.md b/README.md index 673c994..d963351 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,86 @@ # fastwc -A extremely fast wc replacement. \ No newline at end of file +**The word counter GNU wishes it could be.** + +GNU coreutils `wc` is the Apple of the FOSS world. Forty years of +accumulated bureaucracy wrapped in a binary. Translation teams. gettext. +`--help` output in fourteen languages. An autotools contraption the size +of a small city, all so you can count newlines. And when it can't keep +up, it doesn't get faster — it gets *more dependencies*. + +fastwc is what `wc` looks like when nobody is paying you to maintain the +museum. One file. One purpose. No translators. No gnulib. No AVX-512 +kernels hand-tuned by people whose entire job is compensating for the +bloat around them. Just counting, correctly, at full speed. + +## The scoreboard + +The benchmark suite in `benchmarks/` races fastwc against GNU `wc` +(and busybox, if you keep such things installed) — fail-fast. The moment +we are slower, or disagree on a single count, it writes a shame report +and exits non-zero. These are the facts: + +| Suite | Result | +|-------|--------| +| words (6 cases) | **6/6 wins.** Never slower, never wrong. | +| lines (up to 100k lines) | **Wins.** GNU never sees us coming. | +| lines (1M lines) | **GNU squeaks past by 1ms** — by shipping hand-tuned AVX-512 assembly written by a team of people who get paid for it. We call that cheating. Our SIMD pass is coming, and it will not be subtle. | +| lines (10M lines) | Not yet run. The benchmark aborts at the first loss. Coward. | + +The moment fastwc is slower than GNU `wc`, this project has failed and +you should say so loudly in an issue. The benchmark is the contract. + +## Why + +- **GNU wc is a dependency museum.** Its build needs gettext, gnulib, + and a translator for every language on Earth. fastwc needs `cc`. +- **GNU wc is slow where it should be fast.** Counting bytes is not + supposed to be an architectural achievement. +- **GNU wc counts like it's 1985** — because it is. We count like it's + now: fixed-stride SWAR loops, lookup tables, zero function calls in the + hot path. + +## What it does + +``` +fastwc [-lwc] [-m] [file...] +``` + +- `-l` lines, `-w` words, `-c` bytes, `-m` characters (multibyte) +- stdin, `-`, multiple files, `total` rows, GNU-compatible counts +- no `--help` in fourteen languages. One `--help`, in English, the + language of people who ship software + +## Build + +Requires a C compiler and autotools. That's it. No gettext. No gnulib. +No translators. + +```sh +./autogen.sh # autoreconf -fi && ./configure +make +make release # installs the release binary to bin/release/fastwc +``` + +## Benchmark + +```sh +make bench # build release + run every suite +./benchmarks/bench-coreutils.sh # the real fight +./benchmarks/bench-busybox.sh # if you must +``` + +The suites interleave runs so both commands see identical cache warmth, +keep the minimum, and fail the moment fastwc loses a single case. GNU +`wc` is used as an oracle the same way you'd use a broken clock: +occasionally it's right, and it's the only one around. + +## License + +MIT. Do whatever you want. We're not GNU, we won't sue you — we'll just +be faster. + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) — the rules are the deal. +See [STYLEGUIDE.md](STYLEGUIDE.md) — the style is the law. diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md new file mode 100644 index 0000000..7ced74f --- /dev/null +++ b/STYLEGUIDE.md @@ -0,0 +1,92 @@ +# fastwc Style Guide + +The style is the law. GNU wc can afford to look like a committee +designed it — it was. We're one file, one purpose, one opinion. + +## The language + +- **C99 with POSIX.** `_POSIX_C_SOURCE 200809L`. We target the + standard, not a compiler vendor's mood. +- **Zero warnings** with `-Wall -Wextra`. A warning is a bug you + haven't fixed yet. Fix it. +- **No compiler-specific extensions** unless they're guarded and the + benchmark gets faster because of it. `__builtin_popcountll` is the + one exception — it's the price of admission on every compiler that + matters, and the benchmark proves it. + +## Format + +- **4 spaces.** Not tabs, not 8 spaces. GNU uses tabs because GNU was + written in 1985 by people who had strong opinions about typewriters. +- **80 columns.** If your line is longer, your idea is longer than it + needs to be. +- **Braces on the next line** (Allman). Everywhere. Functions, blocks, + switch cases. Consistency is the only religion here, and we're devout. +- **Snake case for everything.** `count_stream`, `ws_tab`, `nread`. + No CamelCase — that's for languages that need help knowing what a + word is. + +## Structure + +- **One file: `src/main.c`.** See CONTRIBUTING.md rule 6. This is not a + suggestion. +- **Everything is `static`.** The binary exports nothing. We're not a + library, we're a verdict. +- **Types end in `_t`.** `counts_t`, and not much else. GNU has a type + for everything and a committee to name it. We have counts, and we + count. +- **No globals except what's truly process-wide.** `flags` and `ws_tab` + live at file scope because they're the program's identity. Everything + else gets passed around like it's hot. + +## The hot path + +`count_stream()`, `count_newlines()`, `count_words()` are the product. +The rest of the file is just the packaging. + +- **Fixed stride.** No per-token function calls. No `isspace()` in a + loop — that's what the lookup table is for. +- **No allocation, no locks, no syscalls in the counting loop.** The + `fread` is the only syscall, and it's not yours to add to. +- **Branchless where it costs nothing.** A predictable branch is fine; + a mispredicted one is a lie you told the CPU. +- **The buffer is `static`, 128 KiB, and never grows.** GNU's wc reads + in chunks too — ours just doesn't make a ceremony of it. + +## Counts + +- **`long long` for everything that counts.** Files can be bigger than + your opinions. Signed, because overflow should scream, not wrap. +- **Lines are `'\n'`s.** Not "lines of text," not "records." A file + without a trailing newline has fewer newlines than it has lines, and + that's correct. GNU agrees with us on this one, so it must be right. +- **Words are whitespace-delimited runs.** The lookup table decides + what whitespace is; the popcount decides how many words; you don't + get an opinion. + +## Errors + +- **`fastwc: : ` on stderr**, and keep going. GNU wc + fails gracefully across multiple files, and we'll credit them that + one thing. Then we'll be faster at it. +- **Exit 1 if anything failed**, 0 otherwise. No drama, no + `EXIT_FAILURE` poetry. + +## Comments + +- **Only when the code can't speak for itself.** The SWAR newline + counter in `count_newlines()` gets a comment because + `(x ^ nl) & cl` is a sentence in a language you don't speak yet. + `i++` does not get a comment. +- **No commented-out code.** The dead don't get to live in the file. +- **No TODO without a date.** A TODO without a date is a promise you + plan to break. + +## What the style is not + +- It is not GNU's style. GNU's style was written by a committee, + ratified by a foundation, and translated into fourteen languages. +- It is not "whatever your editor defaults to." Your editor has + opinions. So do we. Ours are right. +- It is not negotiable in review. If a PR violates the style, the PR + is a rewrite request with extra steps. diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..29ded5d --- /dev/null +++ b/autogen.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh +# Generate the autotools build system and configure an in-tree build. +set -e + +autoreconf -fi +./configure "$@" diff --git a/benchmarks/bench-busybox.sh b/benchmarks/bench-busybox.sh new file mode 100755 index 0000000..7d6327f --- /dev/null +++ b/benchmarks/bench-busybox.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# bench-busybox.sh — benchmark the busybox wc applet against the release +# build of fastwc. Fails fast: the moment fastwc is slower than (or +# disagrees with) busybox wc, a human readable report is written to +# FAILED-benchmark.txt and this script exits non-zero. +# +# usage: ./bench-busybox.sh +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=std.sh +source "$SCRIPT_DIR/std.sh" + +checkfastwc +checkwc busybox +BENCH_NAME="busybox" + +run_benchmark_suite diff --git a/benchmarks/bench-coreutils.sh b/benchmarks/bench-coreutils.sh new file mode 100755 index 0000000..ab3bf2c --- /dev/null +++ b/benchmarks/bench-coreutils.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# bench-coreutils.sh — benchmark the GNU Coreutils wc implementation against +# the release build of fastwc. Fails fast: the moment fastwc is slower than +# (or disagrees with) coreutils wc, a human readable report is written to +# FAILED-benchmark.txt and this script exits non-zero. +# +# usage: ./bench-coreutils.sh +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=std.sh +source "$SCRIPT_DIR/std.sh" + +checkfastwc +checkwc coreutils +BENCH_NAME="coreutils" + +run_benchmark_suite diff --git a/benchmarks/std.sh b/benchmarks/std.sh new file mode 100755 index 0000000..9e88d9c --- /dev/null +++ b/benchmarks/std.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +# +# std.sh — shared "standard library" for the fastwc benchmark scripts. +# +# Provides the helpers every benchmark script needs: +# checkfastwc() verify a release build of fastwc exists +# checkwc() locate the coreutils or busybox wc implementation +# createrandstr() print one random 10-character alphanumeric string +# createtxt() create (or reuse) a text file with N such lines +# run_benchmark_suite() run every word/line case for the selected wc +# +# Source this file from a benchmark script, then: +# checkfastwc +# checkwc coreutils # or: checkwc busybox +# BENCH_NAME="coreutils" +# run_benchmark_suite +# +# The suite fails fast: the moment fastwc is slower than (or disagrees +# with) the selected wc implementation, it writes a human readable report +# to FAILED-benchmark.txt and returns non-zero. + +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FASTWC="$SCRIPT_DIR/../bin/release/fastwc" +DATA_DIR="$SCRIPT_DIR/.data" +GENFILE="$SCRIPT_DIR/tools/genfile" # optional C helper, built by test-all.sh + +BENCH_NAME="${BENCH_NAME:-wc}" # set by the caller: coreutils | busybox +BENCH_REPS="${BENCH_REPS:-3}" # interleaved runs per case; minimum is kept +RESULT_ROWS="" # accumulated results table + +WC_CMD=() # filled by checkwc(), e.g. (wc) or (busybox wc) +TEXT_FILE="" # filled by createtxt() + +if [[ -z "$BENCH_REPS" || "$BENCH_REPS" -lt 1 ]]; then + BENCH_REPS=1 +fi + +# checkfastwc — make sure ../bin/release/fastwc exists and is executable. +# Mimics autotools configure: prints "checking for ... yes/no" and bails +# out with a helpful message when the release build is missing. +checkfastwc() { + printf 'checking for release build fastwc... ' + if [[ -x "$FASTWC" ]]; then + printf 'yes\n' + return 0 + fi + printf 'no\n' + printf 'configure: error: no release build of fastwc found at %s\n' "$FASTWC" >&2 + printf 'configure: error: run "make release" first to generate one\n' >&2 + exit 1 +} + +# checkwc — locate the requested wc implementation and +# store its invocation in $WC_CMD. Mimics autotools configure output and +# exits on failure. +checkwc() { + local impl="$1" + case "$impl" in + coreutils) + printf 'checking for coreutils wc... ' + if command -v wc >/dev/null 2>&1 \ + && wc --version 2>/dev/null | head -n1 | grep -qi 'GNU coreutils'; then + printf 'yes\n' + WC_CMD=(wc) + else + printf 'no\n' + printf 'configure: error: GNU Coreutils wc not found in PATH\n' >&2 + exit 1 + fi + ;; + busybox) + printf 'checking for busybox wc... ' + if command -v busybox >/dev/null 2>&1 \ + && busybox --list 2>/dev/null | grep -qx 'wc'; then + printf 'yes\n' + WC_CMD=(busybox wc) + else + printf 'no\n' + printf 'configure: error: busybox (with the wc applet) not found in PATH\n' >&2 + exit 1 + fi + ;; + *) + printf 'checkwc: error: unknown implementation "%s" (expected coreutils or busybox)\n' "$impl" >&2 + exit 1 + ;; + esac +} + +# createrandstr — print one 10-character random alphanumeric combination. +createrandstr() { + local chars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + local out='' i + for ((i = 0; i < 10; i++)); do + out+="${chars:$((RANDOM % ${#chars})):1}" + done + printf '%s\n' "$out" +} + +# createtxt — make sure a text file with rows of random +# 10-character alphanumeric strings exists. A copy generated by a previous +# run is reused (checked by exact byte size: 10 chars + '\n' per line), so +# repeated benchmark runs are cheap. Prints the path and sets $TEXT_FILE. +createtxt() { + local lines="$1" + local expect=$((lines * 11)) + local have=0 + + TEXT_FILE="$DATA_DIR/words-$lines.txt" + + if [[ -f "$TEXT_FILE" ]]; then + have=$(stat -c '%s' "$TEXT_FILE" 2>/dev/null || printf '0') + fi + + if [[ "$have" -ne "$expect" ]]; then + mkdir -p "$DATA_DIR" + if [[ -x "$GENFILE" ]]; then + "$GENFILE" "$lines" > "$TEXT_FILE" || { + printf 'createtxt: error: failed to generate %s\n' "$TEXT_FILE" >&2 + return 1 + } + else + printf 'createtxt: warning: %s not built, using slow shell fallback\n' "$GENFILE" >&2 + printf 'createtxt: warning: run ./test-all.sh to build the helper tools\n' >&2 + : > "$TEXT_FILE" + for ((i = 0; i < lines; i++)); do + createrandstr >> "$TEXT_FILE" + done + fi + fi + + printf '%s\n' "$TEXT_FILE" +} + +# time_ms — run a command once and print elapsed wall time in ms. +time_ms() { + local s e + s=$(date +%s%N) + "$@" >/dev/null 2>&1 + e=$(date +%s%N) + printf '%s\n' "$(( (e - s) / 1000000 ))" +} + +# capture_count — print the first whitespace-separated field of a +# command's output, i.e. the count reported by `wc -w/-l` or `fastwc -w/-l`. +capture_count() { + "$@" 2>/dev/null | awk 'NR == 1 { print $1 }' +} + +# write_failed_report