huntedbytheirs/fastwc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: skip the multibyte decoder unless a high byte actually appears

Browse Source
This commit is contained in:
huntedbytheirs
2026-09-09 16:10:15 -04:00
parent 0bb53e9c15
commit e5c6c8d3f0
1 changed files with 173 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.c
+173 -29
View File
@@ -384,7 +384,13 @@ typedef struct
} lw_t;
typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws,
int need_lines, int need_words);
int need_lines, int need_words, int need_high,
int *high);
/* need_high: probe the buffer for any byte >= 0x80. When one is seen the
* kernel sets *high and returns immediately with whatever partial counts
* it has; callers that asked for the probe treat the counts as invalid and
* re-run through the multibyte decoder. Pure-ASCII buffers never trigger,
* so the byte-path counts stay valid. */
/*
* Word separators match GNU wc (the benchmark oracle): the locale's
@@ -396,7 +402,7 @@ typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws,
__attribute__((target("avx512f,avx512bw"))) static lw_t
count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
int need_words)
int need_words, int need_high, int *high)
{
const __m512i nl = _mm512_set1_epi8('\n');
const __m512i sp = _mm512_set1_epi8(' ');
@@ -412,6 +418,13 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
__m512i v = _mm512_loadu_si512((const void *)(s + i));
uint64_t nl_mask = 0;
if (need_high && _mm512_movepi8_mask(v))
{
*high = 1;
lw_t r = {lines, words};
return r;
}
if (need_lines)
{
nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl);
@@ -435,6 +448,12 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
for (; i < n; i++)
{
if (need_high && s[i] & 0x80)
{
*high = 1;
lw_t r = {lines, words};
return r;
}
int ws = ws_tab[s[i]];
if (need_lines)
lines += s[i] == '\n';
@@ -453,7 +472,7 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
__attribute__((target("avx2"))) static lw_t
count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
int need_words)
int need_words, int need_high, int *high)
{
const __m256i nl = _mm256_set1_epi8('\n');
const __m256i sp = _mm256_set1_epi8(' ');
@@ -469,6 +488,13 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
__m256i v = _mm256_loadu_si256((const void *)(s + i));
uint32_t nl_mask = 0;
if (need_high && _mm256_movemask_epi8(v))
{
*high = 1;
lw_t r = {lines, words};
return r;
}
if (need_lines)
{
nl_mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl));
@@ -494,6 +520,12 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
for (; i < n; i++)
{
if (need_high && s[i] & 0x80)
{
*high = 1;
lw_t r = {lines, words};
return r;
}
int ws = ws_tab[s[i]];
if (need_lines)
lines += s[i] == '\n';
@@ -512,7 +544,7 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
__attribute__((target("sse2"))) static lw_t
count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
int need_words)
int need_words, int need_high, int *high)
{
const __m128i nl = _mm_set1_epi8('\n');
const __m128i sp = _mm_set1_epi8(' ');
@@ -528,6 +560,13 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
__m128i v = _mm_loadu_si128((const void *)(s + i));
uint32_t nl_mask = 0;
if (need_high && _mm_movemask_epi8(v))
{
*high = 1;
lw_t r = {lines, words};
return r;
}
if (need_lines)
{
nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl));
@@ -550,6 +589,12 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
for (; i < n; i++)
{
if (need_high && s[i] & 0x80)
{
*high = 1;
lw_t r = {lines, words};
return r;
}
int ws = ws_tab[s[i]];
if (need_lines)
lines += s[i] == '\n';
@@ -570,10 +615,22 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
/* Reference path: the two scalar SWAR counters, kept as the fallback. */
static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws,
int need_lines, int need_words)
int need_lines, int need_words, int need_high,
int *high)
{
lw_t r;
if (need_high)
{
for (size_t i = 0; i < n; i++)
if (s[i] & 0x80)
{
*high = 1;
r.lines = 0;
r.words = 0;
return r;
}
}
r.lines = need_lines ? count_newlines(s, n) : 0;
r.words = need_words ? count_words(s, n, prev_ws) : 0;
return r;
@@ -581,7 +638,8 @@ static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws,
static count_lw_fn count_lw = count_lw_scalar; /* chosen by pick_kernel() */
static void count_mapped(const unsigned char *p, size_t n, counts_t *c);
static void count_mapped(const unsigned char *p, size_t n, counts_t *c,
int need_high, int *high);
#if defined(__x86_64__) || defined(__i386__)
static count_lw_fn pick_kernel(void)
@@ -901,12 +959,12 @@ static void count_stream(FILE *fp, counts_t *c)
int fd = fileno(fp);
/* Under a multibyte locale, -w/-m/-L need per-character decoding;
* -l and -c alone stay on the byte path, exactly like GNU wc. */
if ((MB_CUR_MAX > 1) && (flags & (F_CHARS | F_WORDS | F_MAXLEN)))
{
count_stream_mb(fp, c);
return;
}
* -l and -c alone stay on the byte path, exactly like GNU wc.
* mb_semantics is computed once and reused below: regular files are
* first probed on the SIMD byte path (see the has_high fallback),
* while non-regular inputs go straight to the decoder. */
int mb_semantics =
(MB_CUR_MAX > 1) && (flags & (F_CHARS | F_WORDS | F_MAXLEN));
if (debug && using_wc_lines())
debug_methods();
@@ -960,8 +1018,23 @@ static void count_stream(FILE *fp, counts_t *c)
mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (m != MAP_FAILED)
{
count_mapped((const unsigned char *)m, (size_t)st.st_size, c);
munmap(m, (size_t)st.st_size);
counts_t save = *c;
int high = 0;
count_mapped((const unsigned char *)m, (size_t)st.st_size, c,
mb_semantics, &high);
if (mb_semantics && high)
{
/* Non-ASCII input: the byte-path counts are invalid
* under a multibyte locale. Discard them and decode.
* The stream is still positioned at 0 (mmap never
* advanced it), so re-reading counts the whole file. */
*c = save;
munmap(m, (size_t)st.st_size);
count_stream_mb(fp, c);
}
else
munmap(m, (size_t)st.st_size);
if (ferror(fp))
c->ok = 0;
return;
@@ -969,6 +1042,13 @@ static void count_stream(FILE *fp, counts_t *c)
}
}
/* Non-regular input: no mmap probe happened, so decode directly. */
if (mb_semantics)
{
count_stream_mb(fp, c);
return;
}
for (;;)
{
nread = fread(buf, 1, sizeof buf, fp); /* NOLINT: EOF-state FP */
@@ -980,7 +1060,7 @@ static void count_stream(FILE *fp, counts_t *c)
if (flags & (F_LINES | F_WORDS))
{
lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0,
(flags & F_WORDS) != 0);
(flags & F_WORDS) != 0, 0, NULL);
if (flags & F_LINES)
c->lines += r.lines;
if (flags & F_WORDS)
@@ -1008,6 +1088,8 @@ typedef struct
int prev_ws;
int need_lines;
int need_words;
int need_high;
int high;
lw_t r;
} mjob_t;
@@ -1015,7 +1097,8 @@ static void *map_worker(void *arg)
{
mjob_t *j = arg;
j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words);
j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words,
j->need_high, &j->high);
return NULL;
}
@@ -1026,8 +1109,8 @@ static void *map_worker(void *arg)
* which makes the split exact. The kernels are pure, so no locks.
*/
static void count_sliced(const unsigned char *p, size_t n, int nt,
int need_lines, int need_words, long long *lines,
long long *words)
int need_lines, int need_words, int need_high,
int *high, long long *lines, long long *words)
{
mjob_t jobs[MAX_THREADS];
pthread_t th[MAX_THREADS];
@@ -1051,6 +1134,8 @@ static void count_sliced(const unsigned char *p, size_t n, int nt,
jobs[i].prev_ws = (i == 0 || jobs[i].n == 0) ? 1 : ws_tab[p[start - 1]];
jobs[i].need_lines = need_lines;
jobs[i].need_words = need_words;
jobs[i].need_high = need_high;
jobs[i].high = 0;
th[i] = 0;
if (jobs[i].n > 0)
{
@@ -1072,6 +1157,14 @@ static void count_sliced(const unsigned char *p, size_t n, int nt,
tw += jobs[i].r.words;
}
*high = 0;
for (i = 0; i < nt; i++)
if (jobs[i].high)
{
*high = 1;
break;
}
*lines = tl;
*words = tw;
}
@@ -1096,7 +1189,8 @@ static int pick_threads(size_t n)
return nt;
}
static void count_mapped(const unsigned char *p, size_t n, counts_t *c)
static void count_mapped(const unsigned char *p, size_t n, counts_t *c,
int need_high, int *high)
{
int need_lines = (flags & F_LINES) != 0;
int need_words = (flags & F_WORDS) != 0;
@@ -1110,14 +1204,20 @@ static void count_mapped(const unsigned char *p, size_t n, counts_t *c)
if (nt <= 1)
{
int prev_ws = 1;
lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words);
lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words, need_high,
high);
if (need_high && *high)
return; /* partial counts; caller re-decodes */
lines = r.lines;
words = r.words;
}
else
{
count_sliced(p, n, nt, need_lines, need_words, &lines, &words);
count_sliced(p, n, nt, need_lines, need_words, need_high, high,
&lines, &words);
if (need_high && *high)
return; /* partial counts; caller re-decodes */
}
if (flags & F_LINES)
@@ -1786,7 +1886,8 @@ static long long ref_words(const unsigned char *s, size_t n, int *prev_ws)
* the reference instead of only on real AVX-512 hardware.
*/
static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
int *prev_ws, int need_lines, int need_words)
int *prev_ws, int need_lines, int need_words,
int need_high, int *high)
{
long long lines = 0, words = 0;
size_t i = 0;
@@ -1800,6 +1901,12 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
unsigned char c = s[i + j];
uint64_t bit = (uint64_t)1 << j;
if (need_high && (c & 0x80))
{
*high = 1;
lw_t r = {lines, words};
return r;
}
if (c == '\n')
nl_mask |= bit;
if (c == '\n' || c == ' ' || (nbsp_ws && c == 0xa0) ||
@@ -1818,6 +1925,12 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
for (; i < n; i++)
{
int ws = ws_tab[s[i]];
if (need_high && (s[i] & 0x80))
{
*high = 1;
lw_t r = {lines, words};
return r;
}
if (need_lines)
lines += s[i] == '\n';
if (need_words)
@@ -1850,7 +1963,7 @@ static int check_kernel(const char *name, count_lw_fn fn)
for (int nw = 0; nw <= 1; nw++)
{
int a = pw, b = pw;
lw_t got = fn(buf, n, &a, nl, nw);
lw_t got = fn(buf, n, &a, nl, nw, 0, NULL);
long long want_l = nl ? ref_lines(buf, n) : 0;
long long want_w = nw ? ref_words(buf, n, &b) : 0;
@@ -1884,7 +1997,7 @@ static int check_kernel(const char *name, count_lw_fn fn)
for (int pw = 0; pw <= 1; pw++)
{
int a = pw, b = pw;
lw_t got = fn(buf, n, &a, 1, 1);
lw_t got = fn(buf, n, &a, 1, 1, 0, NULL);
if (got.lines != ref_lines(buf, n) ||
got.words != ref_words(buf, n, &b) || a != b)
@@ -1907,7 +2020,7 @@ static int check_kernel(const char *name, count_lw_fn fn)
for (k = 0; k < 256; k++)
buf[k] = pass == 0 ? (unsigned char)k : (unsigned char)(255 - k);
int a = 1, b = 1;
lw_t got = fn(buf, 256, &a, 1, 1);
lw_t got = fn(buf, 256, &a, 1, 1, 0, NULL);
if (got.lines != ref_lines(buf, 256) ||
got.words != ref_words(buf, 256, &b) || a != b)
@@ -1917,6 +2030,35 @@ static int check_kernel(const char *name, count_lw_fn fn)
}
}
/* high-byte probe contract: *high set iff some byte >= 0x80 exists.
* Partial counts on a probe trip are undefined, so only the flag is
* checked here (the caller re-runs through the mb decoder). */
for (size_t probe = 0; probe < 2; probe++)
{
int high = 0;
memset(buf, 'x', 256);
buf[probe == 0 ? 0 : 255] = probe == 0 ? 0x80 : 0xff;
int a = 1;
fn(buf, 256, &a, 1, 1, 1, &high);
if (!high)
{
printf("%s: high probe pos=%zu not detected\n", name, probe);
fails++;
}
memset(buf, 'x', 256);
buf[255] = '\n';
high = 0;
a = 1;
fn(buf, 256, &a, 1, 1, 1, &high);
if (high)
{
printf("%s: ascii probe false-trip\n", name);
fails++;
}
}
printf("%s: %s\n", name, fails ? "FAIL" : "ok");
return fails;
}
@@ -1942,12 +2084,13 @@ static int check_sliced(void)
{
int nt = tcs[ti];
long long tl = 0, tw = 0;
int dh = 0;
long long want_l = count_newlines(buf, n);
int pw = 1;
long long want_w = count_words(buf, n, &pw);
count_sliced(buf, n, nt, 1, 1, &tl, &tw);
if (tl != want_l || tw != want_w)
count_sliced(buf, n, nt, 1, 1, 0, &dh, &tl, &tw);
if (tl != want_l || tw != want_w || dh != 0)
{
printf("sliced: n=%zu nt=%d lines %lld/%lld "
"words %lld/%lld\n",
@@ -1968,12 +2111,13 @@ static int check_sliced(void)
{
int nt = tcs[ti];
long long tl = 0, tw = 0;
int dh = 0;
long long want_l = count_newlines(buf, n);
int pw = 1;
long long want_w = count_words(buf, n, &pw);
count_sliced(buf, n, nt, 1, 1, &tl, &tw);
if (tl != want_l || tw != want_w)
count_sliced(buf, n, nt, 1, 1, 0, &dh, &tl, &tw);
if (tl != want_l || tw != want_w || dh != 0)
{
printf("sliced-ws: n=%zu nt=%d lines %lld/%lld "
"words %lld/%lld\n",