diff --git a/src/main.c b/src/main.c index 283e113..fe85faf 100644 --- a/src/main.c +++ b/src/main.c @@ -384,7 +384,13 @@ typedef struct } lw_t; typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws, - int need_lines, int need_words); + int need_lines, int need_words, int need_high, + int *high); +/* need_high: probe the buffer for any byte >= 0x80. When one is seen the + * kernel sets *high and returns immediately with whatever partial counts + * it has; callers that asked for the probe treat the counts as invalid and + * re-run through the multibyte decoder. Pure-ASCII buffers never trigger, + * so the byte-path counts stay valid. */ /* * Word separators match GNU wc (the benchmark oracle): the locale's @@ -396,7 +402,7 @@ typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws, __attribute__((target("avx512f,avx512bw"))) static lw_t count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines, - int need_words) + int need_words, int need_high, int *high) { const __m512i nl = _mm512_set1_epi8('\n'); const __m512i sp = _mm512_set1_epi8(' '); @@ -412,6 +418,13 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines, __m512i v = _mm512_loadu_si512((const void *)(s + i)); uint64_t nl_mask = 0; + if (need_high && _mm512_movepi8_mask(v)) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } + if (need_lines) { nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl); @@ -435,6 +448,12 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines, for (; i < n; i++) { + if (need_high && s[i] & 0x80) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } int ws = ws_tab[s[i]]; if (need_lines) lines += s[i] == '\n'; @@ -453,7 +472,7 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines, __attribute__((target("avx2"))) static lw_t count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, - int need_words) + int need_words, int need_high, int *high) { const __m256i nl = _mm256_set1_epi8('\n'); const __m256i sp = _mm256_set1_epi8(' '); @@ -469,6 +488,13 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, __m256i v = _mm256_loadu_si256((const void *)(s + i)); uint32_t nl_mask = 0; + if (need_high && _mm256_movemask_epi8(v)) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } + if (need_lines) { nl_mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl)); @@ -494,6 +520,12 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, for (; i < n; i++) { + if (need_high && s[i] & 0x80) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } int ws = ws_tab[s[i]]; if (need_lines) lines += s[i] == '\n'; @@ -512,7 +544,7 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, __attribute__((target("sse2"))) static lw_t count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, - int need_words) + int need_words, int need_high, int *high) { const __m128i nl = _mm_set1_epi8('\n'); const __m128i sp = _mm_set1_epi8(' '); @@ -528,6 +560,13 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, __m128i v = _mm_loadu_si128((const void *)(s + i)); uint32_t nl_mask = 0; + if (need_high && _mm_movemask_epi8(v)) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } + if (need_lines) { nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl)); @@ -550,6 +589,12 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, for (; i < n; i++) { + if (need_high && s[i] & 0x80) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } int ws = ws_tab[s[i]]; if (need_lines) lines += s[i] == '\n'; @@ -570,10 +615,22 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, /* Reference path: the two scalar SWAR counters, kept as the fallback. */ static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws, - int need_lines, int need_words) + int need_lines, int need_words, int need_high, + int *high) { lw_t r; + if (need_high) + { + for (size_t i = 0; i < n; i++) + if (s[i] & 0x80) + { + *high = 1; + r.lines = 0; + r.words = 0; + return r; + } + } r.lines = need_lines ? count_newlines(s, n) : 0; r.words = need_words ? count_words(s, n, prev_ws) : 0; return r; @@ -581,7 +638,8 @@ static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws, static count_lw_fn count_lw = count_lw_scalar; /* chosen by pick_kernel() */ -static void count_mapped(const unsigned char *p, size_t n, counts_t *c); +static void count_mapped(const unsigned char *p, size_t n, counts_t *c, + int need_high, int *high); #if defined(__x86_64__) || defined(__i386__) static count_lw_fn pick_kernel(void) @@ -901,12 +959,12 @@ static void count_stream(FILE *fp, counts_t *c) int fd = fileno(fp); /* Under a multibyte locale, -w/-m/-L need per-character decoding; - * -l and -c alone stay on the byte path, exactly like GNU wc. */ - if ((MB_CUR_MAX > 1) && (flags & (F_CHARS | F_WORDS | F_MAXLEN))) - { - count_stream_mb(fp, c); - return; - } + * -l and -c alone stay on the byte path, exactly like GNU wc. + * mb_semantics is computed once and reused below: regular files are + * first probed on the SIMD byte path (see the has_high fallback), + * while non-regular inputs go straight to the decoder. */ + int mb_semantics = + (MB_CUR_MAX > 1) && (flags & (F_CHARS | F_WORDS | F_MAXLEN)); if (debug && using_wc_lines()) debug_methods(); @@ -960,8 +1018,23 @@ static void count_stream(FILE *fp, counts_t *c) mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (m != MAP_FAILED) { - count_mapped((const unsigned char *)m, (size_t)st.st_size, c); - munmap(m, (size_t)st.st_size); + counts_t save = *c; + int high = 0; + + count_mapped((const unsigned char *)m, (size_t)st.st_size, c, + mb_semantics, &high); + if (mb_semantics && high) + { + /* Non-ASCII input: the byte-path counts are invalid + * under a multibyte locale. Discard them and decode. + * The stream is still positioned at 0 (mmap never + * advanced it), so re-reading counts the whole file. */ + *c = save; + munmap(m, (size_t)st.st_size); + count_stream_mb(fp, c); + } + else + munmap(m, (size_t)st.st_size); if (ferror(fp)) c->ok = 0; return; @@ -969,6 +1042,13 @@ static void count_stream(FILE *fp, counts_t *c) } } + /* Non-regular input: no mmap probe happened, so decode directly. */ + if (mb_semantics) + { + count_stream_mb(fp, c); + return; + } + for (;;) { nread = fread(buf, 1, sizeof buf, fp); /* NOLINT: EOF-state FP */ @@ -980,7 +1060,7 @@ static void count_stream(FILE *fp, counts_t *c) if (flags & (F_LINES | F_WORDS)) { lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0, - (flags & F_WORDS) != 0); + (flags & F_WORDS) != 0, 0, NULL); if (flags & F_LINES) c->lines += r.lines; if (flags & F_WORDS) @@ -1008,6 +1088,8 @@ typedef struct int prev_ws; int need_lines; int need_words; + int need_high; + int high; lw_t r; } mjob_t; @@ -1015,7 +1097,8 @@ static void *map_worker(void *arg) { mjob_t *j = arg; - j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words); + j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words, + j->need_high, &j->high); return NULL; } @@ -1026,8 +1109,8 @@ static void *map_worker(void *arg) * which makes the split exact. The kernels are pure, so no locks. */ static void count_sliced(const unsigned char *p, size_t n, int nt, - int need_lines, int need_words, long long *lines, - long long *words) + int need_lines, int need_words, int need_high, + int *high, long long *lines, long long *words) { mjob_t jobs[MAX_THREADS]; pthread_t th[MAX_THREADS]; @@ -1051,6 +1134,8 @@ static void count_sliced(const unsigned char *p, size_t n, int nt, jobs[i].prev_ws = (i == 0 || jobs[i].n == 0) ? 1 : ws_tab[p[start - 1]]; jobs[i].need_lines = need_lines; jobs[i].need_words = need_words; + jobs[i].need_high = need_high; + jobs[i].high = 0; th[i] = 0; if (jobs[i].n > 0) { @@ -1072,6 +1157,14 @@ static void count_sliced(const unsigned char *p, size_t n, int nt, tw += jobs[i].r.words; } + *high = 0; + for (i = 0; i < nt; i++) + if (jobs[i].high) + { + *high = 1; + break; + } + *lines = tl; *words = tw; } @@ -1096,7 +1189,8 @@ static int pick_threads(size_t n) return nt; } -static void count_mapped(const unsigned char *p, size_t n, counts_t *c) +static void count_mapped(const unsigned char *p, size_t n, counts_t *c, + int need_high, int *high) { int need_lines = (flags & F_LINES) != 0; int need_words = (flags & F_WORDS) != 0; @@ -1110,14 +1204,20 @@ static void count_mapped(const unsigned char *p, size_t n, counts_t *c) if (nt <= 1) { int prev_ws = 1; - lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words); + lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words, need_high, + high); + if (need_high && *high) + return; /* partial counts; caller re-decodes */ lines = r.lines; words = r.words; } else { - count_sliced(p, n, nt, need_lines, need_words, &lines, &words); + count_sliced(p, n, nt, need_lines, need_words, need_high, high, + &lines, &words); + if (need_high && *high) + return; /* partial counts; caller re-decodes */ } if (flags & F_LINES) @@ -1786,7 +1886,8 @@ static long long ref_words(const unsigned char *s, size_t n, int *prev_ws) * the reference instead of only on real AVX-512 hardware. */ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n, - int *prev_ws, int need_lines, int need_words) + int *prev_ws, int need_lines, int need_words, + int need_high, int *high) { long long lines = 0, words = 0; size_t i = 0; @@ -1800,6 +1901,12 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n, unsigned char c = s[i + j]; uint64_t bit = (uint64_t)1 << j; + if (need_high && (c & 0x80)) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } if (c == '\n') nl_mask |= bit; if (c == '\n' || c == ' ' || (nbsp_ws && c == 0xa0) || @@ -1818,6 +1925,12 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n, for (; i < n; i++) { int ws = ws_tab[s[i]]; + if (need_high && (s[i] & 0x80)) + { + *high = 1; + lw_t r = {lines, words}; + return r; + } if (need_lines) lines += s[i] == '\n'; if (need_words) @@ -1850,7 +1963,7 @@ static int check_kernel(const char *name, count_lw_fn fn) for (int nw = 0; nw <= 1; nw++) { int a = pw, b = pw; - lw_t got = fn(buf, n, &a, nl, nw); + lw_t got = fn(buf, n, &a, nl, nw, 0, NULL); long long want_l = nl ? ref_lines(buf, n) : 0; long long want_w = nw ? ref_words(buf, n, &b) : 0; @@ -1884,7 +1997,7 @@ static int check_kernel(const char *name, count_lw_fn fn) for (int pw = 0; pw <= 1; pw++) { int a = pw, b = pw; - lw_t got = fn(buf, n, &a, 1, 1); + lw_t got = fn(buf, n, &a, 1, 1, 0, NULL); if (got.lines != ref_lines(buf, n) || got.words != ref_words(buf, n, &b) || a != b) @@ -1907,7 +2020,7 @@ static int check_kernel(const char *name, count_lw_fn fn) for (k = 0; k < 256; k++) buf[k] = pass == 0 ? (unsigned char)k : (unsigned char)(255 - k); int a = 1, b = 1; - lw_t got = fn(buf, 256, &a, 1, 1); + lw_t got = fn(buf, 256, &a, 1, 1, 0, NULL); if (got.lines != ref_lines(buf, 256) || got.words != ref_words(buf, 256, &b) || a != b) @@ -1917,6 +2030,35 @@ static int check_kernel(const char *name, count_lw_fn fn) } } + /* high-byte probe contract: *high set iff some byte >= 0x80 exists. + * Partial counts on a probe trip are undefined, so only the flag is + * checked here (the caller re-runs through the mb decoder). */ + for (size_t probe = 0; probe < 2; probe++) + { + int high = 0; + + memset(buf, 'x', 256); + buf[probe == 0 ? 0 : 255] = probe == 0 ? 0x80 : 0xff; + int a = 1; + fn(buf, 256, &a, 1, 1, 1, &high); + if (!high) + { + printf("%s: high probe pos=%zu not detected\n", name, probe); + fails++; + } + + memset(buf, 'x', 256); + buf[255] = '\n'; + high = 0; + a = 1; + fn(buf, 256, &a, 1, 1, 1, &high); + if (high) + { + printf("%s: ascii probe false-trip\n", name); + fails++; + } + } + printf("%s: %s\n", name, fails ? "FAIL" : "ok"); return fails; } @@ -1942,12 +2084,13 @@ static int check_sliced(void) { int nt = tcs[ti]; long long tl = 0, tw = 0; + int dh = 0; long long want_l = count_newlines(buf, n); int pw = 1; long long want_w = count_words(buf, n, &pw); - count_sliced(buf, n, nt, 1, 1, &tl, &tw); - if (tl != want_l || tw != want_w) + count_sliced(buf, n, nt, 1, 1, 0, &dh, &tl, &tw); + if (tl != want_l || tw != want_w || dh != 0) { printf("sliced: n=%zu nt=%d lines %lld/%lld " "words %lld/%lld\n", @@ -1968,12 +2111,13 @@ static int check_sliced(void) { int nt = tcs[ti]; long long tl = 0, tw = 0; + int dh = 0; long long want_l = count_newlines(buf, n); int pw = 1; long long want_w = count_words(buf, n, &pw); - count_sliced(buf, n, nt, 1, 1, &tl, &tw); - if (tl != want_l || tw != want_w) + count_sliced(buf, n, nt, 1, 1, 0, &dh, &tl, &tw); + if (tl != want_l || tw != want_w || dh != 0) { printf("sliced-ws: n=%zu nt=%d lines %lld/%lld " "words %lld/%lld\n",