huntedbytheirs
af61660c42
count_sliced computed the slice size from the requested thread count and only then clamped nt to MAX_THREADS, so the last slice ended before the file tail: raising the cap (as pick_threads now does) silently dropped the tail from every count. Cap nt first, then derive per. Monsters (>=256 MiB) now get one thread per core (capped at 24) instead of a hard 16, which measures ~1.17x on the 1B-line case and is flat on warm files. check_sliced now sweeps the cap boundary (16, 24, 25).
1213 lines
34 KiB
C
1213 lines
34 KiB
C
/*
|
|
* fastwc - a fast wc replacement.
|
|
*
|
|
* Fast path (-l/-w/-c): SIMD kernels with runtime dispatch. Each kernel
|
|
* derives the newline mask and the whitespace mask from one load, so
|
|
* lines and words share a single pass over the buffer. AVX-512 when the
|
|
* CPU has it, then AVX-2, then SSE2, then the scalar SWAR reference.
|
|
*/
|
|
|
|
#define _POSIX_C_SOURCE 200809L
|
|
|
|
#include <ctype.h>
|
|
#include <errno.h>
|
|
#include <fcntl.h>
|
|
#include <locale.h>
|
|
#include <pthread.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
#include <wchar.h>
|
|
#include <wctype.h>
|
|
|
|
#if defined(__x86_64__) || defined(__i386__)
|
|
#include <immintrin.h>
|
|
#endif
|
|
|
|
enum
|
|
{
|
|
F_LINES = 1 << 0, /* -l: count '\n' */
|
|
F_WORDS = 1 << 1, /* -w: whitespace-separated tokens */
|
|
F_CHARS = 1 << 2, /* -m: multibyte characters */
|
|
F_BYTES = 1 << 3, /* -c: bytes */
|
|
};
|
|
|
|
enum
|
|
{
|
|
MAX_THREADS = 24, /* pick_threads() and count_sliced() agree on this */
|
|
};
|
|
|
|
static int flags = 0;
|
|
|
|
typedef struct
|
|
{
|
|
long long lines;
|
|
long long words;
|
|
long long chars;
|
|
long long bytes;
|
|
int ok; /* read succeeded */
|
|
} counts_t;
|
|
|
|
static unsigned char ws_tab[256]; /* ws_tab[c] = 1 if c separates words */
|
|
|
|
/*
|
|
* Word separators match GNU wc (the benchmark oracle): the six C-locale
|
|
* isspace() bytes plus U+00A0 (non-breaking space), which GNU counts
|
|
* unless POSIXLY_CORRECT is set.
|
|
*/
|
|
static void init_ws_tab(void)
|
|
{
|
|
for (int i = 0; i < 256; i++)
|
|
ws_tab[i] = (isspace((unsigned char)i) || i == 0xa0) ? 1 : 0;
|
|
}
|
|
|
|
static void usage(FILE *out)
|
|
{
|
|
fprintf(out, "usage: fastwc [-lwc] [-m] [file...]\n"
|
|
"\n"
|
|
"Count lines, words, and bytes (default) or selected counts.\n"
|
|
"With no file, or when file is -, read standard input.\n"
|
|
"\n"
|
|
" -l count lines\n"
|
|
" -w count words\n"
|
|
" -c count bytes\n"
|
|
" -m count characters\n"
|
|
" --help display this help and exit\n"
|
|
" --version output version information and exit\n");
|
|
}
|
|
|
|
/*
|
|
* Count '\n' in fixed 8-byte SWAR chunks. XOR turns '\n' bytes into zero
|
|
* bytes, the borrow-free haszero() flags them, popcount sums them. The
|
|
* common (x - 0x01..) & ~x & 0x80.. trick is NOT used: its borrow chain
|
|
* falsely flags a 0x01 byte that follows a zero byte, which is fine for
|
|
* strlen (lowest set bit) but overcounts here.
|
|
*/
|
|
static long long count_newlines(const unsigned char *s, size_t n)
|
|
{
|
|
const uint64_t nl = 0x0a0a0a0a0a0a0a0aULL;
|
|
const uint64_t seven = 0x7f7f7f7f7f7f7f7fULL;
|
|
long long k = 0;
|
|
size_t i = 0;
|
|
|
|
for (; i + 8 <= n; i += 8)
|
|
{
|
|
uint64_t x;
|
|
memcpy(&x, s + i, 8);
|
|
x ^= nl;
|
|
x = ~(((x & seven) + seven) | x | seven);
|
|
k += (long long)__builtin_popcountll(x);
|
|
}
|
|
for (; i < n; i++)
|
|
k += s[i] == '\n';
|
|
return k;
|
|
}
|
|
|
|
/*
|
|
* Count word starts (whitespace -> non-whitespace transitions) 8 bytes at
|
|
* a time. For a chunk, build a bitmask where bit j = 1 if byte j is
|
|
* whitespace; word starts inside the chunk are the 1->0 transitions of
|
|
* that mask, plus one for the left edge if the previous byte was
|
|
* whitespace. *prev_ws carries the boundary across chunks.
|
|
*/
|
|
static long long count_words(const unsigned char *s, size_t n, int *prev_ws)
|
|
{
|
|
long long w = 0;
|
|
size_t i = 0;
|
|
int prev = *prev_ws;
|
|
|
|
for (; i + 8 <= n; i += 8)
|
|
{
|
|
uint8_t m = 0;
|
|
m |= (uint8_t)ws_tab[s[i + 0]] << 0;
|
|
m |= (uint8_t)ws_tab[s[i + 1]] << 1;
|
|
m |= (uint8_t)ws_tab[s[i + 2]] << 2;
|
|
m |= (uint8_t)ws_tab[s[i + 3]] << 3;
|
|
m |= (uint8_t)ws_tab[s[i + 4]] << 4;
|
|
m |= (uint8_t)ws_tab[s[i + 5]] << 5;
|
|
m |= (uint8_t)ws_tab[s[i + 6]] << 6;
|
|
m |= (uint8_t)ws_tab[s[i + 7]] << 7;
|
|
|
|
/* bit j set iff byte j-1 was whitespace and byte j is not */
|
|
w += (long long)__builtin_popcount((unsigned)((uint8_t)~m & (m << 1)));
|
|
if (prev && !(m & 1))
|
|
w++;
|
|
prev = (m >> 7) & 1;
|
|
}
|
|
|
|
for (; i < n; i++)
|
|
{
|
|
int ws = ws_tab[s[i]];
|
|
if (prev && !ws)
|
|
w++;
|
|
prev = ws;
|
|
}
|
|
|
|
*prev_ws = prev;
|
|
return w;
|
|
}
|
|
|
|
#if defined(__x86_64__) || defined(__i386__)
|
|
/* SSE2 predates POPCNT; count 16-bit masks with the classic bit trick. */
|
|
static unsigned popcount16(unsigned x)
|
|
{
|
|
x = x - ((x >> 1) & 0x5555);
|
|
x = (x & 0x3333) + ((x >> 2) & 0x3333);
|
|
x = (x + (x >> 4)) & 0x0f0f;
|
|
return (x + (x >> 8)) & 0xff;
|
|
}
|
|
#endif /* x86 */
|
|
|
|
typedef struct
|
|
{
|
|
long long lines;
|
|
long long words;
|
|
} lw_t;
|
|
|
|
typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws,
|
|
int need_lines, int need_words);
|
|
|
|
/*
|
|
* Word separators match GNU wc (the benchmark oracle): the six C-locale
|
|
* isspace() bytes plus U+00A0 (non-breaking space). The 0x09..0x0d range
|
|
* is one unsigned compare, (x - 9) < 5, plus equalities for ' ' and NBSP.
|
|
*/
|
|
#if defined(__x86_64__) || defined(__i386__)
|
|
|
|
__attribute__((target("avx512f,avx512bw"))) static lw_t
|
|
count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
|
|
int need_words)
|
|
{
|
|
const __m512i nl = _mm512_set1_epi8('\n');
|
|
const __m512i sp = _mm512_set1_epi8(' ');
|
|
const __m512i nb = _mm512_set1_epi8((char)0xa0);
|
|
const __m512i lo = _mm512_set1_epi8(9);
|
|
const __m512i four = _mm512_set1_epi8(4);
|
|
long long lines = 0, words = 0;
|
|
size_t i = 0;
|
|
uint64_t prev = (*prev_ws != 0);
|
|
|
|
for (; i + 64 <= n; i += 64)
|
|
{
|
|
__m512i v = _mm512_loadu_si512((const void *)(s + i));
|
|
uint64_t nl_mask = 0;
|
|
|
|
if (need_lines)
|
|
{
|
|
nl_mask = (uint64_t)_mm512_cmpeq_epi8_mask(v, nl);
|
|
lines += (long long)_mm_popcnt_u64(nl_mask);
|
|
}
|
|
|
|
if (need_words)
|
|
{
|
|
/* min(d, 4) == d <=> (x - 9) < 5 unsigned; the range
|
|
* already covers '\n', so no newline compare is needed */
|
|
__m512i d = _mm512_sub_epi8(v, lo);
|
|
uint64_t ws =
|
|
nl_mask | (uint64_t)_mm512_cmpeq_epi8_mask(v, sp) |
|
|
(uint64_t)_mm512_cmpeq_epi8_mask(v, nb) |
|
|
(uint64_t)_mm512_cmpeq_epi8_mask(_mm512_min_epu8(d, four), d);
|
|
words += (long long)_mm_popcnt_u64(~ws & ((ws << 1) | prev));
|
|
prev = (ws >> 63) & 1;
|
|
}
|
|
}
|
|
|
|
for (; i < n; i++)
|
|
{
|
|
int ws = ws_tab[s[i]];
|
|
if (need_lines)
|
|
lines += s[i] == '\n';
|
|
if (need_words)
|
|
{
|
|
if (prev && !ws)
|
|
words++;
|
|
prev = (uint64_t)ws;
|
|
}
|
|
}
|
|
|
|
*prev_ws = (int)prev;
|
|
lw_t r = {lines, words};
|
|
return r;
|
|
}
|
|
|
|
__attribute__((target("avx2"))) static lw_t
|
|
count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
|
|
int need_words)
|
|
{
|
|
const __m256i nl = _mm256_set1_epi8('\n');
|
|
const __m256i sp = _mm256_set1_epi8(' ');
|
|
const __m256i nb = _mm256_set1_epi8((char)0xa0);
|
|
const __m256i lo = _mm256_set1_epi8(9);
|
|
const __m256i four = _mm256_set1_epi8(4);
|
|
long long lines = 0, words = 0;
|
|
size_t i = 0;
|
|
uint32_t prev = (*prev_ws != 0);
|
|
|
|
for (; i + 32 <= n; i += 32)
|
|
{
|
|
__m256i v = _mm256_loadu_si256((const void *)(s + i));
|
|
uint32_t nl_mask = 0;
|
|
|
|
if (need_lines)
|
|
{
|
|
nl_mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nl));
|
|
lines += (long long)_mm_popcnt_u32(nl_mask);
|
|
}
|
|
|
|
if (need_words)
|
|
{
|
|
/* min(d, 4) == d <=> (x - 9) < 5 unsigned; the range
|
|
* already covers '\n', so no newline compare is needed */
|
|
__m256i d = _mm256_sub_epi8(v, lo);
|
|
uint32_t ws =
|
|
nl_mask |
|
|
(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, sp)) |
|
|
(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nb)) |
|
|
(uint32_t)_mm256_movemask_epi8(
|
|
_mm256_cmpeq_epi8(_mm256_min_epu8(d, four), d));
|
|
words += (long long)_mm_popcnt_u32(~ws & ((ws << 1) | prev));
|
|
prev = (ws >> 31) & 1;
|
|
}
|
|
}
|
|
|
|
for (; i < n; i++)
|
|
{
|
|
int ws = ws_tab[s[i]];
|
|
if (need_lines)
|
|
lines += s[i] == '\n';
|
|
if (need_words)
|
|
{
|
|
if (prev && !ws)
|
|
words++;
|
|
prev = (uint32_t)ws;
|
|
}
|
|
}
|
|
|
|
*prev_ws = (int)prev;
|
|
lw_t r = {lines, words};
|
|
return r;
|
|
}
|
|
|
|
__attribute__((target("sse2"))) static lw_t
|
|
count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines,
|
|
int need_words)
|
|
{
|
|
const __m128i nl = _mm_set1_epi8('\n');
|
|
const __m128i sp = _mm_set1_epi8(' ');
|
|
const __m128i nb = _mm_set1_epi8((char)0xa0);
|
|
const __m128i lo = _mm_set1_epi8(9);
|
|
const __m128i four = _mm_set1_epi8(4);
|
|
long long lines = 0, words = 0;
|
|
size_t i = 0;
|
|
uint32_t prev = (*prev_ws != 0);
|
|
|
|
for (; i + 16 <= n; i += 16)
|
|
{
|
|
__m128i v = _mm_loadu_si128((const void *)(s + i));
|
|
uint32_t nl_mask = 0;
|
|
|
|
if (need_lines)
|
|
{
|
|
nl_mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nl));
|
|
lines += (long long)popcount16(nl_mask);
|
|
}
|
|
|
|
if (need_words)
|
|
{
|
|
__m128i d = _mm_sub_epi8(v, lo);
|
|
uint32_t ws = nl_mask |
|
|
(uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, sp)) |
|
|
(uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nb)) |
|
|
(uint32_t)_mm_movemask_epi8(
|
|
_mm_cmpeq_epi8(_mm_min_epu8(d, four), d));
|
|
words += (long long)popcount16(~ws & ((ws << 1) | prev));
|
|
prev = (ws >> 15) & 1;
|
|
}
|
|
}
|
|
|
|
for (; i < n; i++)
|
|
{
|
|
int ws = ws_tab[s[i]];
|
|
if (need_lines)
|
|
lines += s[i] == '\n';
|
|
if (need_words)
|
|
{
|
|
if (prev && !ws)
|
|
words++;
|
|
prev = (uint32_t)ws;
|
|
}
|
|
}
|
|
|
|
*prev_ws = (int)prev;
|
|
lw_t r = {lines, words};
|
|
return r;
|
|
}
|
|
|
|
#endif /* x86 */
|
|
|
|
/* Reference path: the two scalar SWAR counters, kept as the fallback. */
|
|
static lw_t count_lw_scalar(const unsigned char *s, size_t n, int *prev_ws,
|
|
int need_lines, int need_words)
|
|
{
|
|
lw_t r;
|
|
|
|
r.lines = need_lines ? count_newlines(s, n) : 0;
|
|
r.words = need_words ? count_words(s, n, prev_ws) : 0;
|
|
return r;
|
|
}
|
|
|
|
static count_lw_fn count_lw = count_lw_scalar; /* chosen by pick_kernel() */
|
|
|
|
static void count_mapped(const unsigned char *p, size_t n, counts_t *c);
|
|
|
|
#if defined(__x86_64__) || defined(__i386__)
|
|
static count_lw_fn pick_kernel(void)
|
|
{
|
|
__builtin_cpu_init();
|
|
if (__builtin_cpu_supports("avx512bw"))
|
|
return count_lw_avx512;
|
|
if (__builtin_cpu_supports("avx2"))
|
|
return count_lw_avx2;
|
|
if (__builtin_cpu_supports("sse2"))
|
|
return count_lw_sse2;
|
|
return count_lw_scalar;
|
|
}
|
|
#else
|
|
static count_lw_fn pick_kernel(void)
|
|
{
|
|
return count_lw_scalar;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Word separator exactly as GNU wc defines it: the Unicode white space
|
|
* set. glibc's iswspace covers it incompletely (U+2007 and U+202F are
|
|
* missing), which is why the table is explicit.
|
|
*/
|
|
static int is_wspace(wchar_t wc)
|
|
{
|
|
if (wc >= 0x2000 && wc <= 0x200a) /* en space .. hair space */
|
|
return 1;
|
|
switch (wc)
|
|
{
|
|
case 0x09: /* tab */
|
|
case 0x0a: /* newline */
|
|
case 0x0b: /* vertical tab */
|
|
case 0x0c: /* form feed */
|
|
case 0x0d: /* carriage return */
|
|
case 0x20: /* space */
|
|
case 0xa0: /* no-break space */
|
|
case 0x1680: /* ogham space mark */
|
|
case 0x2028: /* line separator */
|
|
case 0x2029: /* paragraph separator */
|
|
case 0x202f: /* narrow no-break space */
|
|
case 0x205f: /* medium mathematical space */
|
|
case 0x3000: /* ideographic space */
|
|
return 1;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Multibyte (-m) path: decode each character with mbrtowc, carrying
|
|
* incomplete sequences across read boundaries. Only used when -m is
|
|
* requested, so it stays deliberately simple.
|
|
*/
|
|
static void count_stream_mb(FILE *fp, counts_t *c)
|
|
{
|
|
/*
|
|
* GNU wc's read size and pointer accounting are replicated exactly
|
|
* (256 KiB buffer, p + prev decode pointer, p += charbytes), so
|
|
* counts agree byte for byte with the oracle - its boundary
|
|
* artifacts included. Only used when -m is requested.
|
|
*/
|
|
static unsigned char buf[1 << 18];
|
|
mbstate_t st;
|
|
size_t pend = 0;
|
|
int in_shift = 0;
|
|
int prev_ws = 1;
|
|
|
|
memset(&st, 0, sizeof st);
|
|
|
|
for (;;)
|
|
{
|
|
size_t nread = fread(buf + pend, 1, sizeof buf - pend, fp);
|
|
if (nread == 0 && pend == 0)
|
|
break;
|
|
c->bytes += (long long)nread;
|
|
|
|
unsigned char *p = buf;
|
|
unsigned char *plim = buf + pend + nread;
|
|
size_t prev = pend;
|
|
pend = 0;
|
|
|
|
do
|
|
{
|
|
wchar_t wc;
|
|
size_t charbytes;
|
|
|
|
if (!in_shift && *p < 0x80)
|
|
{
|
|
charbytes = 1;
|
|
wc = *p;
|
|
}
|
|
else
|
|
{
|
|
size_t scanbytes = plim - (p + prev);
|
|
size_t r =
|
|
mbrtowc(&wc, (const char *)(p + prev), scanbytes, &st);
|
|
prev = 0;
|
|
|
|
if (scanbytes < r)
|
|
{
|
|
if (r == (size_t)-2 && plim - p < (long)sizeof buf && nread)
|
|
{ /* incomplete: carry over */
|
|
pend = (size_t)(plim - p);
|
|
memmove(buf, p, pend);
|
|
in_shift = 1;
|
|
break;
|
|
}
|
|
/* encoding error: a byte but not a character */
|
|
p++;
|
|
memset(&st, 0, sizeof st);
|
|
in_shift = 0;
|
|
if (prev_ws)
|
|
{
|
|
c->words++;
|
|
prev_ws = 0;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (wc > 0x10ffff || (wc >= 0xd800 && wc <= 0xdfff))
|
|
{ /* out of range: gnulib rejects, glibc does not */
|
|
p++;
|
|
if (prev_ws)
|
|
{
|
|
c->words++;
|
|
prev_ws = 0;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
charbytes = r + (r == 0);
|
|
in_shift = !mbsinit(&st);
|
|
}
|
|
|
|
if (wc == L'\n')
|
|
c->lines++;
|
|
if (is_wspace(wc))
|
|
{
|
|
prev_ws = 1;
|
|
}
|
|
else if (prev_ws)
|
|
{
|
|
c->words++;
|
|
prev_ws = 0;
|
|
}
|
|
c->chars++;
|
|
p += charbytes;
|
|
} while (p < plim);
|
|
}
|
|
|
|
if (ferror(fp))
|
|
c->ok = 0;
|
|
}
|
|
|
|
static void count_stream(FILE *fp, counts_t *c)
|
|
{
|
|
static _Alignas(64) unsigned char buf[1 << 17]; /* 128 KiB, SIMD-aligned */
|
|
size_t nread;
|
|
int prev_ws = 1; /* start of file: as if preceded by whitespace */
|
|
|
|
if (flags & F_CHARS)
|
|
{
|
|
if (MB_CUR_MAX <= 1)
|
|
{ /* single-byte locale: every byte is a character, like GNU wc */
|
|
for (;;)
|
|
{
|
|
nread = fread(buf, 1, sizeof buf, fp); /* NOLINT */
|
|
if (nread == 0)
|
|
break;
|
|
c->bytes += (long long)nread;
|
|
c->chars += (long long)nread;
|
|
if (flags & (F_LINES | F_WORDS))
|
|
{
|
|
lw_t r =
|
|
count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0,
|
|
(flags & F_WORDS) != 0);
|
|
if (flags & F_LINES)
|
|
c->lines += r.lines;
|
|
if (flags & F_WORDS)
|
|
c->words += r.words;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
count_stream_mb(fp, c);
|
|
}
|
|
if (ferror(fp))
|
|
c->ok = 0;
|
|
return;
|
|
}
|
|
|
|
/* Regular file — named or a stdin redirect — map instead of
|
|
* streaming: no copy, and the count can be split across cores. */
|
|
struct stat st;
|
|
if (fstat(fileno(fp), &st) == 0 && S_ISREG(st.st_mode) && st.st_size > 0)
|
|
{
|
|
if (flags == F_BYTES)
|
|
{ /* GNU wc does not read the file either */
|
|
c->bytes = (long long)st.st_size;
|
|
return;
|
|
}
|
|
void *m = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_PRIVATE,
|
|
fileno(fp), 0);
|
|
if (m != MAP_FAILED)
|
|
{
|
|
count_mapped((const unsigned char *)m, (size_t)st.st_size, c);
|
|
munmap(m, (size_t)st.st_size);
|
|
if (ferror(fp))
|
|
c->ok = 0;
|
|
return;
|
|
}
|
|
}
|
|
|
|
for (;;)
|
|
{
|
|
nread = fread(buf, 1, sizeof buf, fp); /* NOLINT: EOF-state FP */
|
|
if (nread == 0)
|
|
break;
|
|
c->bytes += (long long)nread;
|
|
if (flags & (F_LINES | F_WORDS))
|
|
{
|
|
lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0,
|
|
(flags & F_WORDS) != 0);
|
|
if (flags & F_LINES)
|
|
c->lines += r.lines;
|
|
if (flags & F_WORDS)
|
|
c->words += r.words;
|
|
}
|
|
}
|
|
|
|
if (ferror(fp))
|
|
c->ok = 0;
|
|
}
|
|
|
|
typedef struct
|
|
{
|
|
const unsigned char *s;
|
|
size_t n;
|
|
int prev_ws;
|
|
int need_lines;
|
|
int need_words;
|
|
lw_t r;
|
|
} mjob_t;
|
|
|
|
static void *map_worker(void *arg)
|
|
{
|
|
mjob_t *j = arg;
|
|
|
|
j->r = count_lw(j->s, j->n, &j->prev_ws, j->need_lines, j->need_words);
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* Count a whole mapping in nt slices. Slice starts are 64-byte aligned
|
|
* so SIMD loads never straddle into a neighbor's slice; the word
|
|
* boundary between slices is seeded from the byte before the slice,
|
|
* which makes the split exact. The kernels are pure, so no locks.
|
|
*/
|
|
static void count_sliced(const unsigned char *p, size_t n, int nt,
|
|
int need_lines, int need_words, long long *lines,
|
|
long long *words)
|
|
{
|
|
mjob_t jobs[MAX_THREADS];
|
|
pthread_t th[MAX_THREADS];
|
|
long long tl = 0, tw = 0;
|
|
size_t per;
|
|
int i;
|
|
|
|
if (nt > MAX_THREADS)
|
|
nt = MAX_THREADS; /* cap first: per is derived from the real nt */
|
|
per = (n + (size_t)nt - 1) / (size_t)nt;
|
|
per = (per + 63) & ~(size_t)63;
|
|
if (per == 0)
|
|
per = 64;
|
|
|
|
for (i = 0; i < nt; i++)
|
|
{
|
|
size_t start = (size_t)i * per;
|
|
|
|
jobs[i].s = p + start;
|
|
jobs[i].n = (start + per <= n) ? per : (start < n ? n - start : 0);
|
|
jobs[i].prev_ws = (i == 0 || jobs[i].n == 0) ? 1 : ws_tab[p[start - 1]];
|
|
jobs[i].need_lines = need_lines;
|
|
jobs[i].need_words = need_words;
|
|
th[i] = 0;
|
|
if (jobs[i].n > 0)
|
|
{
|
|
if (pthread_create(&th[i], NULL, map_worker, &jobs[i]) != 0)
|
|
{ /* out of threads: count this slice inline rather than lose it */
|
|
map_worker(&jobs[i]);
|
|
tl += jobs[i].r.lines;
|
|
tw += jobs[i].r.words;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < nt; i++)
|
|
{
|
|
if (th[i] == 0)
|
|
continue;
|
|
pthread_join(th[i], NULL);
|
|
tl += jobs[i].r.lines;
|
|
tw += jobs[i].r.words;
|
|
}
|
|
|
|
*lines = tl;
|
|
*words = tw;
|
|
}
|
|
|
|
static int pick_threads(size_t n)
|
|
{
|
|
long ncpu = sysconf(_SC_NPROCESSORS_ONLN);
|
|
int nt;
|
|
|
|
if (ncpu <= 0)
|
|
ncpu = 1;
|
|
if (n >= (size_t)256 << 20)
|
|
nt = (int)ncpu; /* monsters: one thread per core, readahead wins */
|
|
else if (n >= (size_t)32 << 20)
|
|
nt = (int)ncpu < 12 ? (int)ncpu : 12;
|
|
else if (n >= (size_t)8 << 20)
|
|
nt = (int)ncpu < 4 ? (int)ncpu : 4;
|
|
else
|
|
nt = 1;
|
|
if (nt > MAX_THREADS)
|
|
nt = MAX_THREADS;
|
|
return nt;
|
|
}
|
|
|
|
static void count_mapped(const unsigned char *p, size_t n, counts_t *c)
|
|
{
|
|
int need_lines = (flags & F_LINES) != 0;
|
|
int need_words = (flags & F_WORDS) != 0;
|
|
int nt = pick_threads(n);
|
|
long long lines = 0, words = 0;
|
|
|
|
c->bytes += (long long)n;
|
|
|
|
if (nt <= 1)
|
|
{
|
|
int prev_ws = 1;
|
|
lw_t r = count_lw(p, n, &prev_ws, need_lines, need_words);
|
|
|
|
lines = r.lines;
|
|
words = r.words;
|
|
}
|
|
else
|
|
{
|
|
count_sliced(p, n, nt, need_lines, need_words, &lines, &words);
|
|
}
|
|
|
|
if (flags & F_LINES)
|
|
c->lines += lines;
|
|
if (flags & F_WORDS)
|
|
c->words += words;
|
|
}
|
|
|
|
static void count_file(const char *path, counts_t *c)
|
|
{
|
|
FILE *fp;
|
|
|
|
if (strcmp(path, "-") == 0)
|
|
{
|
|
count_stream(stdin, c);
|
|
if (ferror(stdin))
|
|
fprintf(stderr, "fastwc: standard input: read error: %s\n",
|
|
strerror(errno));
|
|
return;
|
|
}
|
|
|
|
fp = fopen(path, "rb");
|
|
if (fp == NULL)
|
|
{
|
|
fprintf(stderr, "fastwc: %s: %s\n", path, strerror(errno));
|
|
c->ok = 0;
|
|
return;
|
|
}
|
|
|
|
/* count_stream maps regular files itself; -c alone skips reading */
|
|
if (flags == F_BYTES)
|
|
{
|
|
struct stat st;
|
|
if (fstat(fileno(fp), &st) == 0 && S_ISREG(st.st_mode))
|
|
{
|
|
c->bytes = (long long)st.st_size;
|
|
fclose(fp);
|
|
return;
|
|
}
|
|
}
|
|
|
|
count_stream(fp, c);
|
|
|
|
if (ferror(fp))
|
|
fprintf(stderr, "fastwc: %s: read error: %s\n", path, strerror(errno));
|
|
|
|
fclose(fp);
|
|
}
|
|
|
|
static int col_width(long long v)
|
|
{
|
|
int w = 1;
|
|
while (v >= 10)
|
|
{
|
|
v /= 10;
|
|
w++;
|
|
}
|
|
return w;
|
|
}
|
|
|
|
static void widen(int *width, long long v, int enabled)
|
|
{
|
|
int w;
|
|
|
|
if (!enabled)
|
|
return;
|
|
w = col_width(v);
|
|
if (w > *width)
|
|
*width = w;
|
|
}
|
|
|
|
#ifndef FASTWC_SELFTEST
|
|
int main(int argc, char **argv)
|
|
{
|
|
counts_t *rows;
|
|
int nfiles = 0;
|
|
int failed = 0;
|
|
int i, a;
|
|
|
|
init_ws_tab();
|
|
count_lw = pick_kernel();
|
|
|
|
for (a = 1; a < argc; a++)
|
|
{
|
|
const char *arg = argv[a];
|
|
|
|
if (arg[0] != '-' || arg[1] == '\0')
|
|
break; /* first file argument */
|
|
if (strcmp(arg, "--") == 0)
|
|
{
|
|
a++;
|
|
break;
|
|
}
|
|
if (strcmp(arg, "--help") == 0)
|
|
{
|
|
usage(stdout);
|
|
return 0;
|
|
}
|
|
if (strcmp(arg, "--version") == 0)
|
|
{
|
|
printf("fastwc 0.1.0\n");
|
|
return 0;
|
|
}
|
|
for (const char *p = arg + 1; *p; p++)
|
|
{
|
|
switch (*p)
|
|
{
|
|
case 'l':
|
|
flags |= F_LINES;
|
|
break;
|
|
case 'w':
|
|
flags |= F_WORDS;
|
|
break;
|
|
case 'c':
|
|
flags |= F_BYTES;
|
|
break;
|
|
case 'm':
|
|
flags |= F_CHARS;
|
|
break;
|
|
default:
|
|
fprintf(stderr, "fastwc: invalid option -- '%c'\n", *p);
|
|
usage(stderr);
|
|
return 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (flags == 0)
|
|
flags = F_LINES | F_WORDS | F_BYTES; /* wc default: -l -w -c */
|
|
|
|
if (flags & F_CHARS)
|
|
setlocale(LC_CTYPE, "");
|
|
|
|
nfiles = argc - a;
|
|
if (nfiles == 0)
|
|
{
|
|
rows = calloc(1, sizeof *rows);
|
|
rows[0].ok = 1;
|
|
count_stream(stdin, &rows[0]);
|
|
if (!rows[0].ok)
|
|
{
|
|
fprintf(stderr, "fastwc: standard input: read error: %s\n",
|
|
strerror(errno));
|
|
failed = 1;
|
|
}
|
|
nfiles = 1;
|
|
}
|
|
else
|
|
{
|
|
rows = calloc((size_t)nfiles, sizeof *rows);
|
|
for (i = 0; i < nfiles; i++)
|
|
{
|
|
rows[i].ok = 1;
|
|
count_file(argv[a + i], &rows[i]);
|
|
if (!rows[i].ok)
|
|
failed = 1;
|
|
}
|
|
}
|
|
|
|
/* Column widths: widest count in each column across rows + total. */
|
|
int wl = 1, ww = 1, wm = 1, wb = 1;
|
|
long long tl = 0, tw = 0, tm = 0, tb = 0;
|
|
|
|
for (i = 0; i < nfiles; i++)
|
|
{
|
|
counts_t *r = &rows[i];
|
|
tl += r->lines;
|
|
tw += r->words;
|
|
tm += r->chars;
|
|
tb += r->bytes;
|
|
widen(&wl, r->lines, flags & F_LINES);
|
|
widen(&ww, r->words, flags & F_WORDS);
|
|
widen(&wm, r->chars, flags & F_CHARS);
|
|
widen(&wb, r->bytes, flags & F_BYTES);
|
|
}
|
|
widen(&wl, tl, flags & F_LINES);
|
|
widen(&ww, tw, flags & F_WORDS);
|
|
widen(&wm, tm, flags & F_CHARS);
|
|
widen(&wb, tb, flags & F_BYTES);
|
|
|
|
for (i = 0; i < nfiles; i++)
|
|
{
|
|
counts_t *r = &rows[i];
|
|
if (flags & F_LINES)
|
|
printf("%*lld ", wl, r->lines);
|
|
if (flags & F_WORDS)
|
|
printf("%*lld ", ww, r->words);
|
|
if (flags & F_CHARS)
|
|
printf("%*lld ", wm, r->chars);
|
|
if (flags & F_BYTES)
|
|
printf("%*lld ", wb, r->bytes);
|
|
if (argc - a > 0)
|
|
printf("%s", argv[a + i]);
|
|
printf("\n");
|
|
}
|
|
|
|
if (argc - a > 1)
|
|
{
|
|
if (flags & F_LINES)
|
|
printf("%*lld ", wl, tl);
|
|
if (flags & F_WORDS)
|
|
printf("%*lld ", ww, tw);
|
|
if (flags & F_CHARS)
|
|
printf("%*lld ", wm, tm);
|
|
if (flags & F_BYTES)
|
|
printf("%*lld ", wb, tb);
|
|
printf("total\n");
|
|
}
|
|
|
|
free(rows);
|
|
return failed ? 1 : 0;
|
|
}
|
|
#else /* FASTWC_SELFTEST */
|
|
|
|
/*
|
|
* Kernel self-test (cc -DFASTWC_SELFTEST): every kernel is checked
|
|
* against the scalar reference over deterministic inputs covering all
|
|
* chunk tail lengths and both carry states. Kernels this host cannot
|
|
* run (e.g. AVX-512) can be exercised under qemu-x86_64 -cpu max.
|
|
*/
|
|
static unsigned long long rng_state = 1;
|
|
|
|
static unsigned rng32(void)
|
|
{
|
|
rng_state = rng_state * 6364136223846793005ULL + 1442695040888963407ULL;
|
|
return (unsigned)(rng_state >> 33);
|
|
}
|
|
|
|
static long long ref_lines(const unsigned char *s, size_t n)
|
|
{
|
|
return count_newlines(s, n);
|
|
}
|
|
|
|
static long long ref_words(const unsigned char *s, size_t n, int *prev_ws)
|
|
{
|
|
return count_words(s, n, prev_ws);
|
|
}
|
|
|
|
/*
|
|
* Scalar mirror of the AVX-512 kernel: identical 64-byte chunking and
|
|
* 64-bit mask arithmetic (bit-63 carry, 64-bit shift/OR, popcnt) for
|
|
* hosts that cannot execute the real zmm code. It reproduces the
|
|
* kernel's exact whitespace expression - newline/space/NBSP equality
|
|
* plus the (x - 9) < 5 unsigned range trick - rather than consulting
|
|
* ws_tab, so a bug in the kernel's expression is caught here against
|
|
* the reference instead of only on real AVX-512 hardware.
|
|
*/
|
|
static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n,
|
|
int *prev_ws, int need_lines, int need_words)
|
|
{
|
|
long long lines = 0, words = 0;
|
|
size_t i = 0;
|
|
uint64_t prev = (*prev_ws != 0);
|
|
|
|
for (; i + 64 <= n; i += 64)
|
|
{
|
|
uint64_t nl_mask = 0, ws = 0;
|
|
for (size_t j = 0; j < 64; j++)
|
|
{
|
|
unsigned char c = s[i + j];
|
|
uint64_t bit = (uint64_t)1 << j;
|
|
|
|
if (c == '\n')
|
|
nl_mask |= bit;
|
|
if (c == '\n' || c == ' ' || c == 0xa0 || (uint8_t)(c - 9) < 5)
|
|
ws |= bit;
|
|
}
|
|
if (need_lines)
|
|
lines += (long long)__builtin_popcountll(nl_mask);
|
|
if (need_words)
|
|
{
|
|
words += (long long)__builtin_popcountll(~ws & ((ws << 1) | prev));
|
|
prev = (ws >> 63) & 1;
|
|
}
|
|
}
|
|
|
|
for (; i < n; i++)
|
|
{
|
|
int ws = ws_tab[s[i]];
|
|
if (need_lines)
|
|
lines += s[i] == '\n';
|
|
if (need_words)
|
|
{
|
|
if (prev && !ws)
|
|
words++;
|
|
prev = (uint64_t)ws;
|
|
}
|
|
}
|
|
|
|
*prev_ws = (int)prev;
|
|
lw_t r = {lines, words};
|
|
return r;
|
|
}
|
|
|
|
static int check_kernel(const char *name, count_lw_fn fn)
|
|
{
|
|
static unsigned char buf[512];
|
|
int fails = 0;
|
|
size_t n, k;
|
|
|
|
for (n = 0; n <= 512; n++)
|
|
{
|
|
for (k = 0; k < n; k++)
|
|
buf[k] = (unsigned char)rng32();
|
|
for (int pw = 0; pw <= 1; pw++)
|
|
{
|
|
for (int nl = 0; nl <= 1; nl++)
|
|
{
|
|
for (int nw = 0; nw <= 1; nw++)
|
|
{
|
|
int a = pw, b = pw;
|
|
lw_t got = fn(buf, n, &a, nl, nw);
|
|
long long want_l = nl ? ref_lines(buf, n) : 0;
|
|
long long want_w = nw ? ref_words(buf, n, &b) : 0;
|
|
|
|
if (got.lines != want_l || got.words != want_w ||
|
|
a != (nw ? b : pw))
|
|
{
|
|
printf("%s: n=%zu pw=%d nl=%d nw=%d lines %lld/%lld "
|
|
"words %lld/%lld state %d/%d\n",
|
|
name, n, pw, nl, nw, got.lines, want_l,
|
|
got.words, want_w, a, b);
|
|
if (n <= 64)
|
|
{
|
|
for (k = 0; k < n; k++)
|
|
printf("%02x", buf[k]);
|
|
printf("\n");
|
|
}
|
|
fails++;
|
|
if (fails > 5)
|
|
return fails;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* whitespace-heavy patterns exercise the boundary logic harder */
|
|
for (n = 1; n <= 200; n++)
|
|
{
|
|
for (k = 0; k < n; k++)
|
|
buf[k] = (unsigned char)" \t\n\v\f\rx"[rng32() % 7];
|
|
for (int pw = 0; pw <= 1; pw++)
|
|
{
|
|
int a = pw, b = pw;
|
|
lw_t got = fn(buf, n, &a, 1, 1);
|
|
|
|
if (got.lines != ref_lines(buf, n) ||
|
|
got.words != ref_words(buf, n, &b) || a != b)
|
|
{
|
|
printf("%s: ws-pattern n=%zu pw=%d failed\n", name, n, pw);
|
|
for (k = 0; k < n; k++)
|
|
printf("%02x", buf[k]);
|
|
printf("\n");
|
|
fails++;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* exhaustive predicate check: every byte value in a 256-byte ramp,
|
|
* forward and reversed, so a divergence between the kernel's
|
|
* whitespace expression and the reference table changes a count */
|
|
for (int pass = 0; pass < 2; pass++)
|
|
{
|
|
for (k = 0; k < 256; k++)
|
|
buf[k] = pass == 0 ? (unsigned char)k : (unsigned char)(255 - k);
|
|
int a = 1, b = 1;
|
|
lw_t got = fn(buf, 256, &a, 1, 1);
|
|
|
|
if (got.lines != ref_lines(buf, 256) ||
|
|
got.words != ref_words(buf, 256, &b) || a != b)
|
|
{
|
|
printf("%s: predicate ramp pass=%d failed\n", name, pass);
|
|
fails++;
|
|
}
|
|
}
|
|
|
|
printf("%s: %s\n", name, fails ? "FAIL" : "ok");
|
|
return fails;
|
|
}
|
|
|
|
/*
|
|
* Slice-path check: the mmap+threads split must agree with the scalar
|
|
* reference for every thread count and many shapes, since word starts
|
|
* at slice boundaries are seeded from the neighboring byte.
|
|
*/
|
|
static int check_sliced(void)
|
|
{
|
|
static unsigned char buf[9001];
|
|
int fails = 0;
|
|
size_t n, k;
|
|
static const int tcs[] = {
|
|
1, 2, 3, 4, 8, 12, 16, MAX_THREADS, MAX_THREADS + 1};
|
|
|
|
for (n = 0; n <= 9000; n += (n < 300 ? 1 : 37))
|
|
{
|
|
for (k = 0; k < n; k++)
|
|
buf[k] = (unsigned char)rng32();
|
|
for (size_t ti = 0; ti < sizeof tcs / sizeof tcs[0]; ti++)
|
|
{
|
|
int nt = tcs[ti];
|
|
long long tl = 0, tw = 0;
|
|
long long want_l = count_newlines(buf, n);
|
|
int pw = 1;
|
|
long long want_w = count_words(buf, n, &pw);
|
|
|
|
count_sliced(buf, n, nt, 1, 1, &tl, &tw);
|
|
if (tl != want_l || tw != want_w)
|
|
{
|
|
printf("sliced: n=%zu nt=%d lines %lld/%lld "
|
|
"words %lld/%lld\n",
|
|
n, nt, tl, want_l, tw, want_w);
|
|
fails++;
|
|
if (fails > 5)
|
|
return fails;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* whitespace-heavy data stresses the slice boundary seeding */
|
|
for (n = 64; n <= 3000; n += 31)
|
|
{
|
|
for (k = 0; k < n; k++)
|
|
buf[k] = (unsigned char)" \t\n\v\f\r\xa0x"[rng32() % 8];
|
|
for (size_t ti = 0; ti < sizeof tcs / sizeof tcs[0]; ti++)
|
|
{
|
|
int nt = tcs[ti];
|
|
long long tl = 0, tw = 0;
|
|
long long want_l = count_newlines(buf, n);
|
|
int pw = 1;
|
|
long long want_w = count_words(buf, n, &pw);
|
|
|
|
count_sliced(buf, n, nt, 1, 1, &tl, &tw);
|
|
if (tl != want_l || tw != want_w)
|
|
{
|
|
printf("sliced-ws: n=%zu nt=%d lines %lld/%lld "
|
|
"words %lld/%lld\n",
|
|
n, nt, tl, want_l, tw, want_w);
|
|
fails++;
|
|
if (fails > 5)
|
|
return fails;
|
|
}
|
|
}
|
|
}
|
|
|
|
printf("sliced: %s\n", fails ? "FAIL" : "ok");
|
|
return fails;
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
int fails = 0;
|
|
const char *force = getenv("FASTWC_SELFTEST_FORCE");
|
|
|
|
init_ws_tab();
|
|
if (force == NULL || strcmp(force, "scalar") == 0)
|
|
fails += check_kernel("scalar", count_lw_scalar);
|
|
#if defined(__x86_64__) || defined(__i386__)
|
|
if (force == NULL || strcmp(force, "sse2") == 0)
|
|
fails += check_kernel("sse2", count_lw_sse2);
|
|
if (force == NULL || strcmp(force, "avx2") == 0)
|
|
fails += check_kernel("avx2", count_lw_avx2);
|
|
if (force == NULL)
|
|
{
|
|
__builtin_cpu_init();
|
|
if (__builtin_cpu_supports("avx512bw"))
|
|
fails += check_kernel("avx512", count_lw_avx512);
|
|
else
|
|
fails += check_kernel("avx512(mirror)", count_lw_avx512_mirror);
|
|
}
|
|
else if (strcmp(force, "avx512") == 0)
|
|
fails += check_kernel("avx512", count_lw_avx512);
|
|
else if (strcmp(force, "mirror") == 0)
|
|
fails += check_kernel("avx512(mirror)", count_lw_avx512_mirror);
|
|
#endif
|
|
count_lw = count_lw_scalar;
|
|
fails += check_sliced();
|
|
#if defined(__x86_64__) || defined(__i386__)
|
|
count_lw = count_lw_avx2;
|
|
fails += check_sliced();
|
|
count_lw = count_lw_sse2;
|
|
fails += check_sliced();
|
|
count_lw = count_lw_avx512_mirror;
|
|
fails += check_sliced();
|
|
#endif
|
|
if (fails)
|
|
{
|
|
printf("selftest FAILED\n");
|
|
return 1;
|
|
}
|
|
printf("selftest passed\n");
|
|
return 0;
|
|
}
|
|
#endif /* FASTWC_SELFTEST */
|