perf: skip the multibyte decoder unless a high byte actually appears

src/main.c

@@ -384,7 +384,13 @@ typedef struct
 } lw_t;
 
 typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws,
-                            int need_lines, int need_words);
+                            int need_lines, int need_words, int need_high,
+                            int *high);
+/* need_high: probe the buffer for any byte >= 0x80. When one is seen the
+ * kernel sets *high and returns immediately with whatever partial counts
+ * it has; callers that asked for the probe treat the counts as invalid and
+ * re-run through the multibyte decoder. Pure-ASCII buffers never trigger,
+ * so the byte-path counts stay valid. */
 
 /*
  * Word separators match GNU wc (the benchmark oracle): the locale's
@@ -396,7 +402,7 @@ typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws,
 
 __attribute__((target("avx512f,avx512bw"))) static lw_t
 count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
-                int need_words)
+                int need_words, int need_high, int *high)
 {
     const __m512i nl = _mm512_set1_epi8('\n');
     const __m512i sp = _mm512_set1_epi8(' ');
@@ -412,6 +418,13 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
         __m512i v = _mm512_loadu_si512((const void *)(s + i));
         uint64_t nl_mask = 0;
 
+        if (need_high && _mm512_movepi8_mask(v))
+        {
+            *high = 1;
+            lw_t r = {lines, words};
+            return r;
+        }
+
         if (need_lines)
         {
             nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl);
@@ -435,6 +448,12 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
 
     for (; i < n; i++)
     {
+        if (need_high && s[i] & 0x80)
+        {
+            *high = 1;
+            lw_t r = {lines, words};
+            return r;
+        }
         int ws = ws_tab[s[i]];
         if (need_lines)
             lines += s[i] == '\n';
@@ -453,7 +472,7 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
 
 __attribute__((target("avx2"))) static lw_t
 count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
-              int need_words)
+              int need_words, int need_high, int *high)
 {
     const __m256i nl = _mm256_set1_epi8('\n');
     const __m256i sp = _mm256_set1_epi8(' ');
@@ -469,6 +488,13 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
         __m256i v = _mm256_loadu_si256((const void *)(s + i));
         uint32_t nl_mask = 0;
 
+        if (need_high && _mm256_movemask_epi8(v))
+        {
+            *high = 1;
+            lw_t r = {lines, words};
+            return r;
+        }
+
         if (need_lines)
         {
             nl_mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl));
@@ -494,6 +520,12 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
 
     for (; i < n; i++)
     {
+        if (need_high && s[i] & 0x80)
+        {
+            *high = 1;
+            lw_t r = {lines, words};
+            return r;
+        }
         int ws = ws_tab[s[i]];
         if (need_lines)
             lines += s[i] == '\n';
@@ -512,7 +544,7 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
 
 __attribute__((target("sse2"))) static lw_t
 count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
-              int need_words)
+              int need_words, int need_high, int *high)
 {
     const __m128i nl = _mm_set1_epi8('\n');
     const __m128i sp = _mm_set1_epi8(' ');
@@ -528,6 +560,13 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
         __m128i v = _mm_loadu_si128((const void *)(s + i));
         uint32_t nl_mask = 0;
 
+        if (need_high && _mm_movemask_epi8(v))
+        {
+            *high = 1;
+            lw_t r = {lines, words};
+            return r;
+        }
+
         if (need_lines)
         {
             nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl));
@@ -550,6 +589,12 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
 
     for (; i < n; i++)
     {
+        if (need_high && s[i] & 0x80)
+        {
+            *high = 1;
+            lw_t r = {lines, words};
+            return r;
+        }
         int ws = ws_tab[s[i]];
         if (need_lines)
             lines += s[i] == '\n';
@@ -570,10 +615,22 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
 
 /* Reference path: the two scalar SWAR counters, kept as the fallback. */
 static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws,
-                            int need_lines, int need_words)
+                            int need_lines, int need_words, int need_high,
+                            int *high)
 {
     lw_t r;
 
+    if (need_high)
+    {
+        for (size_t i = 0; i < n; i++)
+            if (s[i] & 0x80)
+            {
+                *high = 1;
+                r.lines = 0;
+                r.words = 0;
+                return r;
+            }
+    }
     r.lines = need_lines ? count_newlines(s, n) : 0;
     r.words = need_words ? count_words(s, n, prev_ws) : 0;
     return r;
@@ -581,7 +638,8 @@ static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws,
 
 static count_lw_fn count_lw = count_lw_scalar; /* chosen by pick_kernel() */
 
-static void count_mapped(const unsigned char *p, size_t n, counts_t *c);
+static void count_mapped(const unsigned char *p, size_t n, counts_t *c,
+                         int need_high, int *high);
 
 #if defined(__x86_64__) || defined(__i386__)
 static count_lw_fn pick_kernel(void)
@@ -901,12 +959,12 @@ static void count_stream(FILE *fp, counts_t *c)
     int fd = fileno(fp);
 
     /* Under a multibyte locale, -w/-m/-L need per-character decoding;
-     * -l and -c alone stay on the byte path, exactly like GNU wc. */
+     * -l and -c alone stay on the byte path, exactly like GNU wc.
-    if ((MB_CUR_MAX > 1) && (flags & (F_CHARS | F_WORDS | F_MAXLEN)))
+     * mb_semantics is computed once and reused below: regular files are
-    {
+     * first probed on the SIMD byte path (see the has_high fallback),
-        count_stream_mb(fp, c);
+     * while non-regular inputs go straight to the decoder. */
-        return;
+    int mb_semantics =
-    }
+        (MB_CUR_MAX > 1) && (flags & (F_CHARS | F_WORDS | F_MAXLEN));
 
     if (debug && using_wc_lines())
         debug_methods();
@@ -960,7 +1018,22 @@ static void count_stream(FILE *fp, counts_t *c)
                 mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
             if (m != MAP_FAILED)
             {
-                count_mapped((const unsigned char *)m, (size_t)st.st_size, c);
+                counts_t save = *c;
+                int high = 0;
+
+                count_mapped((const unsigned char *)m, (size_t)st.st_size, c,
+                             mb_semantics, &high);
+                if (mb_semantics && high)
+                {
+                    /* Non-ASCII input: the byte-path counts are invalid
+                     * under a multibyte locale. Discard them and decode.
+                     * The stream is still positioned at 0 (mmap never
+                     * advanced it), so re-reading counts the whole file. */
+                    *c = save;
+                    munmap(m, (size_t)st.st_size);
+                    count_stream_mb(fp, c);
+                }
+                else
                     munmap(m, (size_t)st.st_size);
                 if (ferror(fp))
                     c->ok = 0;
@@ -969,6 +1042,13 @@ static void count_stream(FILE *fp, counts_t *c)
         }
     }
 
+    /* Non-regular input: no mmap probe happened, so decode directly. */
+    if (mb_semantics)
+    {
+        count_stream_mb(fp, c);
+        return;
+    }
+
     for (;;)
     {
         nread = fread(buf, 1, sizeof buf, fp); /* NOLINT: EOF-state FP */
@@ -980,7 +1060,7 @@ static void count_stream(FILE *fp, counts_t *c)
         if (flags & (F_LINES | F_WORDS))
         {
             lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0,
-                              (flags & F_WORDS) != 0);
+                              (flags & F_WORDS) != 0, 0, NULL);
             if (flags & F_LINES)
                 c->lines += r.lines;
             if (flags & F_WORDS)
@@ -1008,6 +1088,8 @@ typedef struct
     int prev_ws;
     int need_lines;
     int need_words;
+    int need_high;
+    int high;
     lw_t r;
 } mjob_t;
 
@@ -1015,7 +1097,8 @@ static void *map_worker(void *arg)
 {
     mjob_t *j = arg;
 
-    j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words);
+    j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words,
+                    j->need_high, &j->high);
     return NULL;
 }
 
@@ -1026,8 +1109,8 @@ static void *map_worker(void *arg)
  * which makes the split exact. The kernels are pure, so no locks.
  */
 static void count_sliced(const unsigned char *p, size_t n, int nt,
-                         int need_lines, int need_words, long long *lines,
+                         int need_lines, int need_words, int need_high,
-                         long long *words)
+                         int *high, long long *lines, long long *words)
 {
     mjob_t jobs[MAX_THREADS];
     pthread_t th[MAX_THREADS];
@@ -1051,6 +1134,8 @@ static void count_sliced(const unsigned char *p, size_t n, int nt,
         jobs[i].prev_ws = (i == 0 || jobs[i].n == 0) ? 1 : ws_tab[p[start - 1]];
         jobs[i].need_lines = need_lines;
         jobs[i].need_words = need_words;
+        jobs[i].need_high = need_high;
+        jobs[i].high = 0;
         th[i] = 0;
         if (jobs[i].n > 0)
         {
@@ -1072,6 +1157,14 @@ static void count_sliced(const unsigned char *p, size_t n, int nt,
         tw += jobs[i].r.words;
     }
 
+    *high = 0;
+    for (i = 0; i < nt; i++)
+        if (jobs[i].high)
+        {
+            *high = 1;
+            break;
+        }
+
     *lines = tl;
     *words = tw;
 }
@@ -1096,7 +1189,8 @@ static int pick_threads(size_t n)
     return nt;
 }
 
-static void count_mapped(const unsigned char *p, size_t n, counts_t *c)
+static void count_mapped(const unsigned char *p, size_t n, counts_t *c,
+                         int need_high, int *high)
 {
     int need_lines = (flags & F_LINES) != 0;
     int need_words = (flags & F_WORDS) != 0;
@@ -1110,14 +1204,20 @@ static void count_mapped(const unsigned char *p, size_t n, counts_t *c)
     if (nt <= 1)
     {
         int prev_ws = 1;
-        lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words);
+        lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words, need_high,
+                          high);
 
+        if (need_high && *high)
+            return; /* partial counts; caller re-decodes */
         lines = r.lines;
         words = r.words;
     }
     else
     {
-        count_sliced(p, n, nt, need_lines, need_words, &lines, &words);
+        count_sliced(p, n, nt, need_lines, need_words, need_high, high,
+                     &lines, &words);
+        if (need_high && *high)
+            return; /* partial counts; caller re-decodes */
     }
 
     if (flags & F_LINES)
@@ -1786,7 +1886,8 @@ static long long ref_words(const unsigned char *s, size_t n, int *prev_ws)
  * the reference instead of only on real AVX-512 hardware.
  */
 static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
-                                   int *prev_ws, int need_lines, int need_words)
+                                   int *prev_ws, int need_lines, int need_words,
+                                   int need_high, int *high)
 {
     long long lines = 0, words = 0;
     size_t i = 0;
@@ -1800,6 +1901,12 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
             unsigned char c = s[i + j];
             uint64_t bit = (uint64_t)1 << j;
 
+            if (need_high && (c & 0x80))
+            {
+                *high = 1;
+                lw_t r = {lines, words};
+                return r;
+            }
             if (c == '\n')
                 nl_mask |= bit;
             if (c == '\n' || c == ' ' || (nbsp_ws && c == 0xa0) ||
@@ -1818,6 +1925,12 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
     for (; i < n; i++)
     {
         int ws = ws_tab[s[i]];
+        if (need_high && (s[i] & 0x80))
+        {
+            *high = 1;
+            lw_t r = {lines, words};
+            return r;
+        }
         if (need_lines)
             lines += s[i] == '\n';
         if (need_words)
@@ -1850,7 +1963,7 @@ static int check_kernel(const char *name, count_lw_fn fn)
                 for (int nw = 0; nw <= 1; nw++)
                 {
                     int a = pw, b = pw;
-                    lw_t got = fn(buf, n, &a, nl, nw);
+                    lw_t got = fn(buf, n, &a, nl, nw, 0, NULL);
                     long long want_l = nl ? ref_lines(buf, n) : 0;
                     long long want_w = nw ? ref_words(buf, n, &b) : 0;
 
@@ -1884,7 +1997,7 @@ static int check_kernel(const char *name, count_lw_fn fn)
         for (int pw = 0; pw <= 1; pw++)
         {
             int a = pw, b = pw;
-            lw_t got = fn(buf, n, &a, 1, 1);
+            lw_t got = fn(buf, n, &a, 1, 1, 0, NULL);
 
             if (got.lines != ref_lines(buf, n) ||
                 got.words != ref_words(buf, n, &b) || a != b)
@@ -1907,7 +2020,7 @@ static int check_kernel(const char *name, count_lw_fn fn)
         for (k = 0; k < 256; k++)
             buf[k] = pass == 0 ? (unsigned char)k : (unsigned char)(255 - k);
         int a = 1, b = 1;
-        lw_t got = fn(buf, 256, &a, 1, 1);
+        lw_t got = fn(buf, 256, &a, 1, 1, 0, NULL);
 
         if (got.lines != ref_lines(buf, 256) ||
             got.words != ref_words(buf, 256, &b) || a != b)
@@ -1917,6 +2030,35 @@ static int check_kernel(const char *name, count_lw_fn fn)
         }
     }
 
+    /* high-byte probe contract: *high set iff some byte >= 0x80 exists.
+     * Partial counts on a probe trip are undefined, so only the flag is
+     * checked here (the caller re-runs through the mb decoder). */
+    for (size_t probe = 0; probe < 2; probe++)
+    {
+        int high = 0;
+
+        memset(buf, 'x', 256);
+        buf[probe == 0 ? 0 : 255] = probe == 0 ? 0x80 : 0xff;
+        int a = 1;
+        fn(buf, 256, &a, 1, 1, 1, &high);
+        if (!high)
+        {
+            printf("%s: high probe pos=%zu not detected\n", name, probe);
+            fails++;
+        }
+
+        memset(buf, 'x', 256);
+        buf[255] = '\n';
+        high = 0;
+        a = 1;
+        fn(buf, 256, &a, 1, 1, 1, &high);
+        if (high)
+        {
+            printf("%s: ascii probe false-trip\n", name);
+            fails++;
+        }
+    }
+
     printf("%s: %s\n", name, fails ? "FAIL" : "ok");
     return fails;
 }
@@ -1942,12 +2084,13 @@ static int check_sliced(void)
         {
             int nt = tcs[ti];
             long long tl = 0, tw = 0;
+            int dh = 0;
             long long want_l = count_newlines(buf, n);
             int pw = 1;
             long long want_w = count_words(buf, n, &pw);
 
-            count_sliced(buf, n, nt, 1, 1, &tl, &tw);
+            count_sliced(buf, n, nt, 1, 1, 0, &dh, &tl, &tw);
-            if (tl != want_l || tw != want_w)
+            if (tl != want_l || tw != want_w || dh != 0)
             {
                 printf("sliced: n=%zu nt=%d lines %lld/%lld "
                        "words %lld/%lld\n",
@@ -1968,12 +2111,13 @@ static int check_sliced(void)
         {
             int nt = tcs[ti];
             long long tl = 0, tw = 0;
+            int dh = 0;
             long long want_l = count_newlines(buf, n);
             int pw = 1;
             long long want_w = count_words(buf, n, &pw);
 
-            count_sliced(buf, n, nt, 1, 1, &tl, &tw);
+            count_sliced(buf, n, nt, 1, 1, 0, &dh, &tl, &tw);
-            if (tl != want_l || tw != want_w)
+            if (tl != want_l || tw != want_w || dh != 0)
             {
                 printf("sliced-ws: n=%zu nt=%d lines %lld/%lld "
                        "words %lld/%lld\n",