diff --git a/README.md b/README.md index 041fffb..fa213f1 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ writes a shame report and exits non-zero. These are the facts: | lines (1M lines) | **Win: 1ms vs 2ms.** GNU's AVX-512 assist can't beat a mapped file. | | lines (10M lines) | **Win: 8-9ms vs 22-24ms (~2.5x).** GNU's lead never survives contact with the buffer. | | lines (100M lines) | **Win: ~70ms vs ~140ms.** The monster race. GNU gets lapped. | -| lines (1B lines) | **Solo, ~6-8s.** 11 GB in one pass; the only bottleneck left is the disk. | +| lines (1B lines) | **Solo, ~4-6s.** 11 GB in one pass; the only bottleneck left is the disk. | | stdin words (1M lines) | **Win: 12x.** GNU still reads stdin like it's 1985. | | stdin lines (10M lines) | **Win: ~2.5x.** We map stdin redirects; GNU maps nothing. | diff --git a/benchmarks/files/lines/bench.sh b/benchmarks/files/lines/bench.sh index 68b9f1e..0c251fe 100755 --- a/benchmarks/files/lines/bench.sh +++ b/benchmarks/files/lines/bench.sh @@ -14,7 +14,7 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" source "$REPO_DIR/benchmarks/std.sh" checkfastwc -checkwc coreutils +checkwc BENCH_NAME="coreutils" printf 'benchmarking %s wc vs fastwc: lines, file input (%s interleaved runs each, minimum kept)\n' \ diff --git a/benchmarks/files/words/bench.sh b/benchmarks/files/words/bench.sh index 5791fcb..e14449a 100755 --- a/benchmarks/files/words/bench.sh +++ b/benchmarks/files/words/bench.sh @@ -12,7 +12,7 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" source "$REPO_DIR/benchmarks/std.sh" checkfastwc -checkwc coreutils +checkwc BENCH_NAME="coreutils" printf 'benchmarking %s wc vs fastwc: words, file input (%s interleaved runs each, minimum kept)\n' \ diff --git a/benchmarks/std.sh b/benchmarks/std.sh index 6e1a142..c505e53 100755 --- a/benchmarks/std.sh +++ b/benchmarks/std.sh @@ -27,6 +27,12 @@ set -u +# Pin the C locale: GNU wc -w silently switches to multibyte decoding under a +# UTF-8 locale, which would slow the oracle down and mask the documented +# byte-semantics divergence. Both sides count bytes here. +export LC_ALL=C +export LC_CTYPE=C + FASTWC="$REPO_DIR/bin/release/fastwc" DATA_DIR="$SCRIPT_DIR/.data" GENFILE="$REPO_DIR/benchmarks/tools/genfile" # optional C helper, built by test-all.sh diff --git a/benchmarks/stdin/piping/bench.sh b/benchmarks/stdin/piping/bench.sh index 5841e70..f8b3b0a 100755 --- a/benchmarks/stdin/piping/bench.sh +++ b/benchmarks/stdin/piping/bench.sh @@ -14,7 +14,7 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" source "$REPO_DIR/benchmarks/std.sh" checkfastwc -checkwc coreutils +checkwc BENCH_NAME="coreutils" printf 'benchmarking %s wc vs fastwc: stdin (%s interleaved runs each, minimum kept)\n' \ diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 59c8f16..a6fb1fe 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -17,12 +17,13 @@ case, so every number below survived contact with the contract. | lines, 1M | 2-3ms | **1ms** | | lines, 10M | 21-24ms | **8-9ms** | | lines, 100M (monster) | ~140ms | **~70ms** | -| lines, 1B (solo) | — | **~6s** | +| lines, 1B (solo) | — | **~4-6s** | | bytes, 1GB sparse | reads all of it | `st_size`, no read | That is a ~2.5x win over GNU on 10M lines, a 2x win on 1M lines, and a 2x win on the 100M monster. At 1B lines — 11 GB of data — the solo -run lands around 6-8 seconds (125-170 Mlines/s), and the bottleneck is +run lands around 4-6 seconds (200-270 Mlines/s, warm cache), and the +bottleneck is honest to admit: an 11 GB file does not fit in the 15 GB of RAM this machine has, so the last monster is racing the disk. The 100M case, which fits, runs at ~17 GB/s, and that number is the counting. @@ -42,11 +43,11 @@ which fits, runs at ~17 GB/s, and that number is the counting. the data comes through stdin, but how we read it is our business. The stdin suite is why this shows up in the scoreboard too. 3. **Parallel across cores.** Files over 8 MiB are split into 64-byte - aligned slices counted by up to 8 threads. The kernels are pure, so - the split needs no locks; word boundaries between slices are seeded - from the byte before the slice, which makes the split exact. Below - 8 MiB the thread spawn would cost more than the counting, so we - don't bother. + aligned slices counted by up to 16 threads (12 past 32 MiB, 16 past + 256 MiB). The kernels are pure, so the split needs no locks; word + boundaries between slices are seeded from the byte before the slice, + which makes the split exact. Below 8 MiB the thread spawn would cost + more than the counting, so we don't bother. 4. **No work that isn't asked for.** `-c` on a regular file is `st_size` from `fstat` — GNU figured that one out too, so we copied the good idea. `-l` without `-w` skips the whitespace mask entirely. diff --git a/src/main.c b/src/main.c index b9ee085..5149e26 100644 --- a/src/main.c +++ b/src/main.c @@ -146,6 +146,7 @@ static long long count_words(const unsigned char *s, size_t n, int *prev_ws) return w; } +#if defined(__x86_64__) || defined(__i386__) /* SSE2 predates POPCNT; count 16-bit masks with the classic bit trick. */ static unsigned popcount16(unsigned x) { @@ -154,6 +155,7 @@ static unsigned popcount16(unsigned x) x = (x + (x >> 4)) & 0x0f0f; return (x + (x >> 8)) & 0xff; } +#endif /* x86 */ typedef struct { @@ -162,7 +164,7 @@ typedef struct } lw_t; typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws, - int need_words); + int need_lines, int need_words); /* * Word separators match GNU wc (the benchmark oracle): the six C-locale @@ -172,7 +174,8 @@ typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws, #if defined(__x86_64__) || defined(__i386__) __attribute__((target("avx512f,avx512bw"))) static lw_t -count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words) +count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines, + int need_words) { const __m512i nl = _mm512_set1_epi8('\n'); const __m512i sp = _mm512_set1_epi8(' '); @@ -186,13 +189,18 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words) for (; i + 64 <= n; i += 64) { __m512i v = _mm512_loadu_si512((const void *)(s + i)); - uint64_t nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl); + uint64_t nl_mask = 0; - lines += (long long)_mm_popcnt_u64(nl_mask); + if (need_lines) + { + nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl); + lines += (long long)_mm_popcnt_u64(nl_mask); + } if (need_words) { - /* min(d, 4) == d <=> (x - 9) < 5 unsigned */ + /* min(d, 4) == d <=> (x - 9) < 5 unsigned; the range + * already covers '\n', so no newline compare is needed */ __m512i d = _mm512_sub_epi8(v, lo); uint64_t ws = nl_mask | (uint64_t)_mm512_cmpeq_epi8_mask(v, sp) | @@ -206,7 +214,8 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words) for (; i < n; i++) { int ws = ws_tab[s[i]]; - lines += s[i] == '\n'; + if (need_lines) + lines += s[i] == '\n'; if (need_words) { if (prev && !ws) @@ -221,7 +230,8 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words) } __attribute__((target("avx2"))) static lw_t -count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words) +count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, + int need_words) { const __m256i nl = _mm256_set1_epi8('\n'); const __m256i sp = _mm256_set1_epi8(' '); @@ -235,14 +245,18 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words) for (; i + 32 <= n; i += 32) { __m256i v = _mm256_loadu_si256((const void *)(s + i)); - uint32_t nl_mask = - (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl)); + uint32_t nl_mask = 0; - lines += (long long)_mm_popcnt_u32(nl_mask); + if (need_lines) + { + nl_mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl)); + lines += (long long)_mm_popcnt_u32(nl_mask); + } if (need_words) { - /* min(d, 4) == d <=> (x - 9) < 5 unsigned */ + /* min(d, 4) == d <=> (x - 9) < 5 unsigned; the range + * already covers '\n', so no newline compare is needed */ __m256i d = _mm256_sub_epi8(v, lo); uint32_t ws = nl_mask | @@ -258,7 +272,8 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words) for (; i < n; i++) { int ws = ws_tab[s[i]]; - lines += s[i] == '\n'; + if (need_lines) + lines += s[i] == '\n'; if (need_words) { if (prev && !ws) @@ -273,7 +288,8 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words) } __attribute__((target("sse2"))) static lw_t -count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words) +count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, + int need_words) { const __m128i nl = _mm_set1_epi8('\n'); const __m128i sp = _mm_set1_epi8(' '); @@ -287,9 +303,13 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words) for (; i + 16 <= n; i += 16) { __m128i v = _mm_loadu_si128((const void *)(s + i)); - uint32_t nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl)); + uint32_t nl_mask = 0; - lines += (long long)popcount16(nl_mask); + if (need_lines) + { + nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl)); + lines += (long long)popcount16(nl_mask); + } if (need_words) { @@ -307,7 +327,8 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words) for (; i < n; i++) { int ws = ws_tab[s[i]]; - lines += s[i] == '\n'; + if (need_lines) + lines += s[i] == '\n'; if (need_words) { if (prev && !ws) @@ -325,11 +346,11 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words) /* Reference path: the two scalar SWAR counters, kept as the fallback. */ static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws, - int need_words) + int need_lines, int need_words) { lw_t r; - r.lines = count_newlines(s, n); + r.lines = need_lines ? count_newlines(s, n) : 0; r.words = need_words ? count_words(s, n, prev_ws) : 0; return r; } @@ -513,7 +534,8 @@ static void count_stream(FILE *fp, counts_t *c) if (flags & (F_LINES | F_WORDS)) { lw_t r = - count_lw(buf, nread, &prev_ws, (flags & F_WORDS) != 0); + count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0, + (flags & F_WORDS) != 0); if (flags & F_LINES) c->lines += r.lines; if (flags & F_WORDS) @@ -560,7 +582,8 @@ static void count_stream(FILE *fp, counts_t *c) c->bytes += (long long)nread; if (flags & (F_LINES | F_WORDS)) { - lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_WORDS) != 0); + lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0, + (flags & F_WORDS) != 0); if (flags & F_LINES) c->lines += r.lines; if (flags & F_WORDS) @@ -577,6 +600,7 @@ typedef struct const unsigned char *s; size_t n; int prev_ws; + int need_lines; int need_words; lw_t r; } mjob_t; @@ -585,7 +609,7 @@ static void *map_worker(void *arg) { mjob_t *j = arg; - j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_words); + j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words); return NULL; } @@ -596,14 +620,21 @@ static void *map_worker(void *arg) * which makes the split exact. The kernels are pure, so no locks. */ static void count_sliced(const unsigned char *p, size_t n, int nt, - int need_words, long long *lines, long long *words) + int need_lines, int need_words, long long *lines, + long long *words) { - mjob_t jobs[8]; - pthread_t th[8]; + enum + { + MAX_THREADS = 16 + }; + mjob_t jobs[MAX_THREADS]; + pthread_t th[MAX_THREADS]; long long tl = 0, tw = 0; size_t per = (n + (size_t)nt - 1) / (size_t)nt; int i; + if (nt > MAX_THREADS) + nt = MAX_THREADS; per = (per + 63) & ~(size_t)63; if (per == 0) per = 64; @@ -615,11 +646,18 @@ static void count_sliced(const unsigned char *p, size_t n, int nt, jobs[i].s = p + start; jobs[i].n = (start + per <= n) ? per : (start < n ? n - start : 0); jobs[i].prev_ws = (i == 0 || jobs[i].n == 0) ? 1 : ws_tab[p[start - 1]]; + jobs[i].need_lines = need_lines; jobs[i].need_words = need_words; + th[i] = 0; if (jobs[i].n > 0) - pthread_create(&th[i], NULL, map_worker, &jobs[i]); - else - th[i] = 0; + { + if (pthread_create(&th[i], NULL, map_worker, &jobs[i]) != 0) + { /* out of threads: count this slice inline rather than lose it */ + map_worker(&jobs[i]); + tl += jobs[i].r.lines; + tw += jobs[i].r.words; + } + } } for (i = 0; i < nt; i++) @@ -640,8 +678,10 @@ static int pick_threads(size_t n) long ncpu = sysconf(_SC_NPROCESSORS_ONLN); int nt; - if (n >= (size_t)32 << 20) - nt = 8; + if (n >= (size_t)256 << 20) + nt = 16; + else if (n >= (size_t)32 << 20) + nt = 12; else if (n >= (size_t)8 << 20) nt = 4; else @@ -653,6 +693,7 @@ static int pick_threads(size_t n) static void count_mapped(const unsigned char *p, size_t n, counts_t *c) { + int need_lines = (flags & F_LINES) != 0; int need_words = (flags & F_WORDS) != 0; int nt = pick_threads(n); long long lines = 0, words = 0; @@ -662,14 +703,14 @@ static void count_mapped(const unsigned char *p, size_t n, counts_t *c) if (nt <= 1) { int prev_ws = 1; - lw_t r = count_lw(p, n, &prev_ws, need_words); + lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words); lines = r.lines; words = r.words; } else { - count_sliced(p, n, nt, need_words, &lines, &words); + count_sliced(p, n, nt, need_lines, need_words, &lines, &words); } if (flags & F_LINES) @@ -915,7 +956,7 @@ static long long ref_words(const unsigned char *s, size_t n, int *prev_ws) * the algorithm; this proves the width. */ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n, - int *prev_ws, int need_words) + int *prev_ws, int need_lines, int need_words) { long long lines = 0, words = 0; size_t i = 0; @@ -931,7 +972,8 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n, if (ws_tab[s[i + j]]) ws |= (uint64_t)1 << j; } - lines += (long long)__builtin_popcountll(nl_mask); + if (need_lines) + lines += (long long)__builtin_popcountll(nl_mask); if (need_words) { words += (long long)__builtin_popcountll(~ws & ((ws << 1) | prev)); @@ -942,7 +984,8 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n, for (; i < n; i++) { int ws = ws_tab[s[i]]; - lines += s[i] == '\n'; + if (need_lines) + lines += s[i] == '\n'; if (need_words) { if (prev && !ws) @@ -968,29 +1011,32 @@ static int check_kernel(const char *name, count_lw_fn fn) buf[k] = (unsigned char)rng32(); for (int pw = 0; pw <= 1; pw++) { - for (int nw = 0; nw <= 1; nw++) + for (int nl = 0; nl <= 1; nl++) { - int a = pw, b = pw; - lw_t got = fn(buf, n, &a, nw); - long long want_l = ref_lines(buf, n); - long long want_w = nw ? ref_words(buf, n, &b) : 0; - - if (got.lines != want_l || got.words != want_w || - a != (nw ? b : pw)) + for (int nw = 0; nw <= 1; nw++) { - printf("%s: n=%zu pw=%d nw=%d lines %lld/%lld " - "words %lld/%lld state %d/%d\n", - name, n, pw, nw, got.lines, want_l, got.words, - want_w, a, b); - if (n <= 64) + int a = pw, b = pw; + lw_t got = fn(buf, n, &a, nl, nw); + long long want_l = nl ? ref_lines(buf, n) : 0; + long long want_w = nw ? ref_words(buf, n, &b) : 0; + + if (got.lines != want_l || got.words != want_w || + a != (nw ? b : pw)) { - for (k = 0; k < n; k++) - printf("%02x", buf[k]); - printf("\n"); + printf("%s: n=%zu pw=%d nl=%d nw=%d lines %lld/%lld " + "words %lld/%lld state %d/%d\n", + name, n, pw, nl, nw, got.lines, want_l, + got.words, want_w, a, b); + if (n <= 64) + { + for (k = 0; k < n; k++) + printf("%02x", buf[k]); + printf("\n"); + } + fails++; + if (fails > 5) + return fails; } - fails++; - if (fails > 5) - return fails; } } } @@ -1004,7 +1050,7 @@ static int check_kernel(const char *name, count_lw_fn fn) for (int pw = 0; pw <= 1; pw++) { int a = pw, b = pw; - lw_t got = fn(buf, n, &a, 1); + lw_t got = fn(buf, n, &a, 1, 1); if (got.lines != ref_lines(buf, n) || got.words != ref_words(buf, n, &b) || a != b) @@ -1045,7 +1091,7 @@ static int check_sliced(void) int pw = 1; long long want_w = count_words(buf, n, &pw); - count_sliced(buf, n, nt, 1, &tl, &tw); + count_sliced(buf, n, nt, 1, 1, &tl, &tw); if (tl != want_l || tw != want_w) { printf("sliced: n=%zu nt=%d lines %lld/%lld " @@ -1070,7 +1116,7 @@ static int check_sliced(void) int pw = 1; long long want_w = count_words(buf, n, &pw); - count_sliced(buf, n, nt, 1, &tl, &tw); + count_sliced(buf, n, nt, 1, 1, &tl, &tw); if (tl != want_l || tw != want_w) { printf("sliced-ws: n=%zu nt=%d lines %lld/%lld "