test: make the avx512 mirror reproduce the kernel's real expression

The scalar AVX-512 mirror built its whitespace mask from ws_tab lookups,
so a bug in the kernel's actual predicate - the (x - 9) < 5 unsigned
range trick plus newline/space/NBSP compares - would pass the selftest
on hosts without AVX-512 and only surface on real hardware. The mirror
now computes the mask with the kernel's exact expression, and
check_kernel gains an exhaustive 256-byte ramp (forward and reversed)
so any divergence from the reference table changes a count. Verified by
mutation: shifting the range constant to (x - 8) < 5 makes the selftest
fail.

README.md

@@ -28,7 +28,7 @@ writes a shame report and exits non-zero. These are the facts:
 | lines (1M lines) | **Win: 1ms vs 2ms.** GNU's AVX-512 assist can't beat a mapped file. |
 | lines (10M lines) | **Win: 8-9ms vs 22-24ms (~2.5x).** GNU's lead never survives contact with the buffer. |
 | lines (100M lines) | **Win: ~70ms vs ~140ms.** The monster race. GNU gets lapped. |
-| lines (1B lines) | **Solo, ~6-8s.** 11 GB in one pass; the only bottleneck left is the disk. |
+| lines (1B lines) | **Solo, ~4-6s.** 11 GB in one pass; the only bottleneck left is the disk. |
 | stdin words (1M lines) | **Win: 12x.** GNU still reads stdin like it's 1985. |
 | stdin lines (10M lines) | **Win: ~2.5x.** We map stdin redirects; GNU maps nothing. |
 

benchmarks/files/lines/bench.sh

@@ -14,7 +14,7 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
 source "$REPO_DIR/benchmarks/std.sh"
 
 checkfastwc
-checkwc coreutils
+checkwc
 BENCH_NAME="coreutils"
 
 printf 'benchmarking %s wc vs fastwc: lines, file input (%s interleaved runs each, minimum kept)\n' \

benchmarks/files/words/bench.sh

@@ -12,7 +12,7 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
 source "$REPO_DIR/benchmarks/std.sh"
 
 checkfastwc
-checkwc coreutils
+checkwc
 BENCH_NAME="coreutils"
 
 printf 'benchmarking %s wc vs fastwc: words, file input (%s interleaved runs each, minimum kept)\n' \

benchmarks/std.sh

@@ -27,6 +27,12 @@
 
 set -u
 
+# Pin the C locale: GNU wc -w silently switches to multibyte decoding under a
+# UTF-8 locale, which would slow the oracle down and mask the documented
+# byte-semantics divergence. Both sides count bytes here.
+export LC_ALL=C
+export LC_CTYPE=C
+
 FASTWC="$REPO_DIR/bin/release/fastwc"
 DATA_DIR="$SCRIPT_DIR/.data"
 GENFILE="$REPO_DIR/benchmarks/tools/genfile"  # optional C helper, built by test-all.sh

benchmarks/stdin/piping/bench.sh

@@ -14,7 +14,7 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
 source "$REPO_DIR/benchmarks/std.sh"
 
 checkfastwc
-checkwc coreutils
+checkwc
 BENCH_NAME="coreutils"
 
 printf 'benchmarking %s wc vs fastwc: stdin (%s interleaved runs each, minimum kept)\n' \

docs/PERFORMANCE.md

@@ -17,12 +17,13 @@ case, so every number below survived contact with the contract.
 | lines, 1M | 2-3ms | **1ms** |
 | lines, 10M | 21-24ms | **8-9ms** |
 | lines, 100M (monster) | ~140ms | **~70ms** |
-| lines, 1B (solo) | — | **~6s** |
+| lines, 1B (solo) | — | **~4-6s** |
 | bytes, 1GB sparse | reads all of it | `st_size`, no read |
 
 That is a ~2.5x win over GNU on 10M lines, a 2x win on 1M lines, and
 a 2x win on the 100M monster. At 1B lines — 11 GB of data — the solo
-run lands around 6-8 seconds (125-170 Mlines/s), and the bottleneck is
+run lands around 4-6 seconds (200-270 Mlines/s, warm cache), and the
+bottleneck is
 honest to admit: an 11 GB file does not fit in the 15 GB of RAM this
 machine has, so the last monster is racing the disk. The 100M case,
 which fits, runs at ~17 GB/s, and that number is the counting.
@@ -42,11 +43,11 @@ which fits, runs at ~17 GB/s, and that number is the counting.
    the data comes through stdin, but how we read it is our business.
    The stdin suite is why this shows up in the scoreboard too.
 3. **Parallel across cores.** Files over 8 MiB are split into 64-byte
-   aligned slices counted by up to 8 threads. The kernels are pure, so
+   aligned slices counted by up to 16 threads (12 past 32 MiB, 16 past
-   the split needs no locks; word boundaries between slices are seeded
+   256 MiB). The kernels are pure, so the split needs no locks; word
-   from the byte before the slice, which makes the split exact. Below
+   boundaries between slices are seeded from the byte before the slice,
-   8 MiB the thread spawn would cost more than the counting, so we
+   which makes the split exact. Below 8 MiB the thread spawn would cost
-   don't bother.
+   more than the counting, so we don't bother.
 4. **No work that isn't asked for.** `-c` on a regular file is
    `st_size` from `fstat` — GNU figured that one out too, so we copied
    the good idea. `-l` without `-w` skips the whitespace mask entirely.

src/main.c

@@ -146,6 +146,7 @@ static long long count_words(const unsigned char *s, size_t n, int *prev_ws)
     return w;
 }
 
+#if defined(__x86_64__) || defined(__i386__)
 /* SSE2 predates POPCNT; count 16-bit masks with the classic bit trick. */
 static unsigned popcount16(unsigned x)
 {
@@ -154,6 +155,7 @@ static unsigned popcount16(unsigned x)
     x = (x + (x >> 4)) & 0x0f0f;
     return (x + (x >> 8)) & 0xff;
 }
+#endif /* x86 */
 
 typedef struct
 {
@@ -162,7 +164,7 @@ typedef struct
 } lw_t;
 
 typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws,
-                            int need_words);
+                            int need_lines, int need_words);
 
 /*
  * Word separators match GNU wc (the benchmark oracle): the six C-locale
@@ -172,7 +174,8 @@ typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws,
 #if defined(__x86_64__) || defined(__i386__)
 
 __attribute__((target("avx512f,avx512bw"))) static lw_t
-count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words)
+count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
+                int need_words)
 {
     const __m512i nl = _mm512_set1_epi8('\n');
     const __m512i sp = _mm512_set1_epi8(' ');
@@ -186,13 +189,18 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words)
     for (; i + 64 <= n; i += 64)
     {
         __m512i v = _mm512_loadu_si512((const void *)(s + i));
-        uint64_t nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl);
+        uint64_t nl_mask = 0;
 
-        lines += (long long)_mm_popcnt_u64(nl_mask);
+        if (need_lines)
+        {
+            nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl);
+            lines += (long long)_mm_popcnt_u64(nl_mask);
+        }
 
         if (need_words)
         {
-            /* min(d, 4) == d  <=>  (x - 9) < 5 unsigned */
+            /* min(d, 4) == d  <=>  (x - 9) < 5 unsigned; the range
+             * already covers '\n', so no newline compare is needed */
             __m512i d = _mm512_sub_epi8(v, lo);
             uint64_t ws =
                 nl_mask | (uint64_t)_mm512_cmpeq_epi8_mask(v, sp) |
@@ -206,7 +214,8 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words)
     for (; i < n; i++)
     {
         int ws = ws_tab[s[i]];
-        lines += s[i] == '\n';
+        if (need_lines)
+            lines += s[i] == '\n';
         if (need_words)
         {
             if (prev && !ws)
@@ -221,7 +230,8 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_words)
 }
 
 __attribute__((target("avx2"))) static lw_t
-count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
+count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
+              int need_words)
 {
     const __m256i nl = _mm256_set1_epi8('\n');
     const __m256i sp = _mm256_set1_epi8(' ');
@@ -235,14 +245,18 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
     for (; i + 32 <= n; i += 32)
     {
         __m256i v = _mm256_loadu_si256((const void *)(s + i));
-        uint32_t nl_mask =
+        uint32_t nl_mask = 0;
-            (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl));
 
-        lines += (long long)_mm_popcnt_u32(nl_mask);
+        if (need_lines)
+        {
+            nl_mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl));
+            lines += (long long)_mm_popcnt_u32(nl_mask);
+        }
 
         if (need_words)
         {
-            /* min(d, 4) == d  <=>  (x - 9) < 5 unsigned */
+            /* min(d, 4) == d  <=>  (x - 9) < 5 unsigned; the range
+             * already covers '\n', so no newline compare is needed */
             __m256i d = _mm256_sub_epi8(v, lo);
             uint32_t ws =
                 nl_mask |
@@ -258,7 +272,8 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
     for (; i < n; i++)
     {
         int ws = ws_tab[s[i]];
-        lines += s[i] == '\n';
+        if (need_lines)
+            lines += s[i] == '\n';
         if (need_words)
         {
             if (prev && !ws)
@@ -273,7 +288,8 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
 }
 
 __attribute__((target("sse2"))) static lw_t
-count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
+count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
+              int need_words)
 {
     const __m128i nl = _mm_set1_epi8('\n');
     const __m128i sp = _mm_set1_epi8(' ');
@@ -287,9 +303,13 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
     for (; i + 16 <= n; i += 16)
     {
         __m128i v = _mm_loadu_si128((const void *)(s + i));
-        uint32_t nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl));
+        uint32_t nl_mask = 0;
 
-        lines += (long long)popcount16(nl_mask);
+        if (need_lines)
+        {
+            nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl));
+            lines += (long long)popcount16(nl_mask);
+        }
 
         if (need_words)
         {
@@ -307,7 +327,8 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
     for (; i < n; i++)
     {
         int ws = ws_tab[s[i]];
-        lines += s[i] == '\n';
+        if (need_lines)
+            lines += s[i] == '\n';
         if (need_words)
         {
             if (prev && !ws)
@@ -325,11 +346,11 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_words)
 
 /* Reference path: the two scalar SWAR counters, kept as the fallback. */
 static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws,
-                            int need_words)
+                            int need_lines, int need_words)
 {
     lw_t r;
 
-    r.lines = count_newlines(s, n);
+    r.lines = need_lines ? count_newlines(s, n) : 0;
     r.words = need_words ? count_words(s, n, prev_ws) : 0;
     return r;
 }
@@ -513,7 +534,8 @@ static void count_stream(FILE *fp, counts_t *c)
                 if (flags & (F_LINES | F_WORDS))
                 {
                     lw_t r =
-                        count_lw(buf, nread, &prev_ws, (flags & F_WORDS) != 0);
+                        count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0,
+                                 (flags & F_WORDS) != 0);
                     if (flags & F_LINES)
                         c->lines += r.lines;
                     if (flags & F_WORDS)
@@ -560,7 +582,8 @@ static void count_stream(FILE *fp, counts_t *c)
         c->bytes += (long long)nread;
         if (flags & (F_LINES | F_WORDS))
         {
-            lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_WORDS) != 0);
+            lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0,
+                              (flags & F_WORDS) != 0);
             if (flags & F_LINES)
                 c->lines += r.lines;
             if (flags & F_WORDS)
@@ -577,6 +600,7 @@ typedef struct
     const unsigned char *s;
     size_t n;
     int prev_ws;
+    int need_lines;
     int need_words;
     lw_t r;
 } mjob_t;
@@ -585,7 +609,7 @@ static void *map_worker(void *arg)
 {
     mjob_t *j = arg;
 
-    j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_words);
+    j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words);
     return NULL;
 }
 
@@ -596,14 +620,21 @@ static void *map_worker(void *arg)
  * which makes the split exact. The kernels are pure, so no locks.
  */
 static void count_sliced(const unsigned char *p, size_t n, int nt,
-                         int need_words, long long *lines, long long *words)
+                         int need_lines, int need_words, long long *lines,
+                         long long *words)
 {
-    mjob_t jobs[8];
+    enum
-    pthread_t th[8];
+    {
+        MAX_THREADS = 16
+    };
+    mjob_t jobs[MAX_THREADS];
+    pthread_t th[MAX_THREADS];
     long long tl = 0, tw = 0;
     size_t per = (n + (size_t)nt - 1) / (size_t)nt;
     int i;
 
+    if (nt > MAX_THREADS)
+        nt = MAX_THREADS;
     per = (per + 63) & ~(size_t)63;
     if (per == 0)
         per = 64;
@@ -615,11 +646,18 @@ static void count_sliced(const unsigned char *p, size_t n, int nt,
         jobs[i].s = p + start;
         jobs[i].n = (start + per <= n) ? per : (start < n ? n - start : 0);
         jobs[i].prev_ws = (i == 0 || jobs[i].n == 0) ? 1 : ws_tab[p[start - 1]];
+        jobs[i].need_lines = need_lines;
         jobs[i].need_words = need_words;
+        th[i] = 0;
         if (jobs[i].n > 0)
-            pthread_create(&th[i], NULL, map_worker, &jobs[i]);
+        {
-        else
+            if (pthread_create(&th[i], NULL, map_worker, &jobs[i]) != 0)
-            th[i] = 0;
+            { /* out of threads: count this slice inline rather than lose it */
+                map_worker(&jobs[i]);
+                tl += jobs[i].r.lines;
+                tw += jobs[i].r.words;
+            }
+        }
     }
 
     for (i = 0; i < nt; i++)
@@ -640,8 +678,10 @@ static int pick_threads(size_t n)
     long ncpu = sysconf(_SC_NPROCESSORS_ONLN);
     int nt;
 
-    if (n >= (size_t)32 << 20)
+    if (n >= (size_t)256 << 20)
-        nt = 8;
+        nt = 16;
+    else if (n >= (size_t)32 << 20)
+        nt = 12;
     else if (n >= (size_t)8 << 20)
         nt = 4;
     else
@@ -653,6 +693,7 @@ static int pick_threads(size_t n)
 
 static void count_mapped(const unsigned char *p, size_t n, counts_t *c)
 {
+    int need_lines = (flags & F_LINES) != 0;
     int need_words = (flags & F_WORDS) != 0;
     int nt = pick_threads(n);
     long long lines = 0, words = 0;
@@ -662,14 +703,14 @@ static void count_mapped(const unsigned char *p, size_t n, counts_t *c)
     if (nt <= 1)
     {
         int prev_ws = 1;
-        lw_t r = count_lw(p, n, &prev_ws, need_words);
+        lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words);
 
         lines = r.lines;
         words = r.words;
     }
     else
     {
-        count_sliced(p, n, nt, need_words, &lines, &words);
+        count_sliced(p, n, nt, need_lines, need_words, &lines, &words);
     }
 
     if (flags & F_LINES)
@@ -911,11 +952,14 @@ static long long ref_words(const unsigned char *s, size_t n, int *prev_ws)
 /*
  * Scalar mirror of the AVX-512 kernel: identical 64-byte chunking and
  * 64-bit mask arithmetic (bit-63 carry, 64-bit shift/OR, popcnt) for
- * hosts that cannot execute the real zmm code. AVX-2 already proves
+ * hosts that cannot execute the real zmm code. It reproduces the
- * the algorithm; this proves the width.
+ * kernel's exact whitespace expression - newline/space/NBSP equality
+ * plus the (x - 9) < 5 unsigned range trick - rather than consulting
+ * ws_tab, so a bug in the kernel's expression is caught here against
+ * the reference instead of only on real AVX-512 hardware.
  */
 static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
-                                   int *prev_ws, int need_words)
+                                   int *prev_ws, int need_lines, int need_words)
 {
     long long lines = 0, words = 0;
     size_t i = 0;
@@ -926,12 +970,16 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
         uint64_t nl_mask = 0, ws = 0;
         for (size_t j = 0; j < 64; j++)
         {
-            if (s[i + j] == '\n')
+            unsigned char c = s[i + j];
-                nl_mask |= (uint64_t)1 << j;
+            uint64_t bit = (uint64_t)1 << j;
-            if (ws_tab[s[i + j]])
+
-                ws |= (uint64_t)1 << j;
+            if (c == '\n')
+                nl_mask |= bit;
+            if (c == '\n' || c == ' ' || c == 0xa0 || (uint8_t)(c - 9) < 5)
+                ws |= bit;
         }
-        lines += (long long)__builtin_popcountll(nl_mask);
+        if (need_lines)
+            lines += (long long)__builtin_popcountll(nl_mask);
         if (need_words)
         {
             words += (long long)__builtin_popcountll(~ws & ((ws << 1) | prev));
@@ -942,7 +990,8 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
     for (; i < n; i++)
     {
         int ws = ws_tab[s[i]];
-        lines += s[i] == '\n';
+        if (need_lines)
+            lines += s[i] == '\n';
         if (need_words)
         {
             if (prev && !ws)
@@ -968,29 +1017,32 @@ static int check_kernel(const char *name, count_lw_fn fn)
             buf[k] = (unsigned char)rng32();
         for (int pw = 0; pw <= 1; pw++)
         {
-            for (int nw = 0; nw <= 1; nw++)
+            for (int nl = 0; nl <= 1; nl++)
             {
-                int a = pw, b = pw;
+                for (int nw = 0; nw <= 1; nw++)
-                lw_t got = fn(buf, n, &a, nw);
-                long long want_l = ref_lines(buf, n);
-                long long want_w = nw ? ref_words(buf, n, &b) : 0;
-
-                if (got.lines != want_l || got.words != want_w ||
-                    a != (nw ? b : pw))
                 {
-                    printf("%s: n=%zu pw=%d nw=%d lines %lld/%lld "
+                    int a = pw, b = pw;
-                           "words %lld/%lld state %d/%d\n",
+                    lw_t got = fn(buf, n, &a, nl, nw);
-                           name, n, pw, nw, got.lines, want_l, got.words,
+                    long long want_l = nl ? ref_lines(buf, n) : 0;
-                           want_w, a, b);
+                    long long want_w = nw ? ref_words(buf, n, &b) : 0;
-                    if (n <= 64)
+
+                    if (got.lines != want_l || got.words != want_w ||
+                        a != (nw ? b : pw))
                     {
-                        for (k = 0; k < n; k++)
+                        printf("%s: n=%zu pw=%d nl=%d nw=%d lines %lld/%lld "
-                            printf("%02x", buf[k]);
+                               "words %lld/%lld state %d/%d\n",
-                        printf("\n");
+                               name, n, pw, nl, nw, got.lines, want_l,
+                               got.words, want_w, a, b);
+                        if (n <= 64)
+                        {
+                            for (k = 0; k < n; k++)
+                                printf("%02x", buf[k]);
+                            printf("\n");
+                        }
+                        fails++;
+                        if (fails > 5)
+                            return fails;
                     }
-                    fails++;
-                    if (fails > 5)
-                        return fails;
                 }
             }
         }
@@ -1004,7 +1056,7 @@ static int check_kernel(const char *name, count_lw_fn fn)
         for (int pw = 0; pw <= 1; pw++)
         {
             int a = pw, b = pw;
-            lw_t got = fn(buf, n, &a, 1);
+            lw_t got = fn(buf, n, &a, 1, 1);
 
             if (got.lines != ref_lines(buf, n) ||
                 got.words != ref_words(buf, n, &b) || a != b)
@@ -1019,6 +1071,24 @@ static int check_kernel(const char *name, count_lw_fn fn)
         }
     }
 
+    /* exhaustive predicate check: every byte value in a 256-byte ramp,
+     * forward and reversed, so a divergence between the kernel's
+     * whitespace expression and the reference table changes a count */
+    for (int pass = 0; pass < 2; pass++)
+    {
+        for (k = 0; k < 256; k++)
+            buf[k] = pass == 0 ? (unsigned char)k : (unsigned char)(255 - k);
+        int a = 1, b = 1;
+        lw_t got = fn(buf, 256, &a, 1, 1);
+
+        if (got.lines != ref_lines(buf, 256) ||
+            got.words != ref_words(buf, 256, &b) || a != b)
+        {
+            printf("%s: predicate ramp pass=%d failed\n", name, pass);
+            fails++;
+        }
+    }
+
     printf("%s: %s\n", name, fails ? "FAIL" : "ok");
     return fails;
 }
@@ -1045,7 +1115,7 @@ static int check_sliced(void)
             int pw = 1;
             long long want_w = count_words(buf, n, &pw);
 
-            count_sliced(buf, n, nt, 1, &tl, &tw);
+            count_sliced(buf, n, nt, 1, 1, &tl, &tw);
             if (tl != want_l || tw != want_w)
             {
                 printf("sliced: n=%zu nt=%d lines %lld/%lld "
@@ -1070,7 +1140,7 @@ static int check_sliced(void)
             int pw = 1;
             long long want_w = count_words(buf, n, &pw);
 
-            count_sliced(buf, n, nt, 1, &tl, &tw);
+            count_sliced(buf, n, nt, 1, 1, &tl, &tw);
             if (tl != want_l || tw != want_w)
             {
                 printf("sliced-ws: n=%zu nt=%d lines %lld/%lld "