fix: cap thread count before slice sizing, scale monsters to ncpu
count_sliced computed the slice size from the requested thread count and only then clamped nt to MAX_THREADS, so the last slice ended before the file tail: raising the cap (as pick_threads now does) silently dropped the tail from every count. Cap nt first, then derive per. Monsters (>=256 MiB) now get one thread per core (capped at 24) instead of a hard 16, which measures ~1.17x on the 1B-line case and is flat on warm files. check_sliced now sweeps the cap boundary (16, 24, 25).
This commit is contained in:
+6
-5
@@ -43,11 +43,12 @@ which fits, runs at ~17 GB/s, and that number is the counting.
|
|||||||
the data comes through stdin, but how we read it is our business.
|
the data comes through stdin, but how we read it is our business.
|
||||||
The stdin suite is why this shows up in the scoreboard too.
|
The stdin suite is why this shows up in the scoreboard too.
|
||||||
3. **Parallel across cores.** Files over 8 MiB are split into 64-byte
|
3. **Parallel across cores.** Files over 8 MiB are split into 64-byte
|
||||||
aligned slices counted by up to 16 threads (12 past 32 MiB, 16 past
|
aligned slices counted by up to one thread per core (capped at 24)
|
||||||
256 MiB). The kernels are pure, so the split needs no locks; word
|
past 256 MiB, 12 past 32 MiB, 4 past 8 MiB. The kernels are pure, so
|
||||||
boundaries between slices are seeded from the byte before the slice,
|
the split needs no locks; word boundaries between slices are seeded
|
||||||
which makes the split exact. Below 8 MiB the thread spawn would cost
|
from the byte before the slice, which makes the split exact. Below
|
||||||
more than the counting, so we don't bother.
|
8 MiB the thread spawn would cost more than the counting, so we don't
|
||||||
|
bother.
|
||||||
4. **No work that isn't asked for.** `-c` on a regular file is
|
4. **No work that isn't asked for.** `-c` on a regular file is
|
||||||
`st_size` from `fstat` — GNU figured that one out too, so we copied
|
`st_size` from `fstat` — GNU figured that one out too, so we copied
|
||||||
the good idea. `-l` without `-w` skips the whitespace mask entirely.
|
the good idea. `-l` without `-w` skips the whitespace mask entirely.
|
||||||
|
|||||||
+21
-13
@@ -36,6 +36,11 @@ enum
|
|||||||
F_BYTES = 1 << 3, /* -c: bytes */
|
F_BYTES = 1 << 3, /* -c: bytes */
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum
|
||||||
|
{
|
||||||
|
MAX_THREADS = 24, /* pick_threads() and count_sliced() agree on this */
|
||||||
|
};
|
||||||
|
|
||||||
static int flags = 0;
|
static int flags = 0;
|
||||||
|
|
||||||
typedef struct
|
typedef struct
|
||||||
@@ -623,18 +628,15 @@ static void count_sliced(const unsigned char *p, size_t n, int nt,
|
|||||||
int need_lines, int need_words, long long *lines,
|
int need_lines, int need_words, long long *lines,
|
||||||
long long *words)
|
long long *words)
|
||||||
{
|
{
|
||||||
enum
|
|
||||||
{
|
|
||||||
MAX_THREADS = 16
|
|
||||||
};
|
|
||||||
mjob_t jobs[MAX_THREADS];
|
mjob_t jobs[MAX_THREADS];
|
||||||
pthread_t th[MAX_THREADS];
|
pthread_t th[MAX_THREADS];
|
||||||
long long tl = 0, tw = 0;
|
long long tl = 0, tw = 0;
|
||||||
size_t per = (n + (size_t)nt - 1) / (size_t)nt;
|
size_t per;
|
||||||
int i;
|
int i;
|
||||||
|
|
||||||
if (nt > MAX_THREADS)
|
if (nt > MAX_THREADS)
|
||||||
nt = MAX_THREADS;
|
nt = MAX_THREADS; /* cap first: per is derived from the real nt */
|
||||||
|
per = (n + (size_t)nt - 1) / (size_t)nt;
|
||||||
per = (per + 63) & ~(size_t)63;
|
per = (per + 63) & ~(size_t)63;
|
||||||
if (per == 0)
|
if (per == 0)
|
||||||
per = 64;
|
per = 64;
|
||||||
@@ -678,16 +680,18 @@ static int pick_threads(size_t n)
|
|||||||
long ncpu = sysconf(_SC_NPROCESSORS_ONLN);
|
long ncpu = sysconf(_SC_NPROCESSORS_ONLN);
|
||||||
int nt;
|
int nt;
|
||||||
|
|
||||||
|
if (ncpu <= 0)
|
||||||
|
ncpu = 1;
|
||||||
if (n >= (size_t)256 << 20)
|
if (n >= (size_t)256 << 20)
|
||||||
nt = 16;
|
nt = (int)ncpu; /* monsters: one thread per core, readahead wins */
|
||||||
else if (n >= (size_t)32 << 20)
|
else if (n >= (size_t)32 << 20)
|
||||||
nt = 12;
|
nt = (int)ncpu < 12 ? (int)ncpu : 12;
|
||||||
else if (n >= (size_t)8 << 20)
|
else if (n >= (size_t)8 << 20)
|
||||||
nt = 4;
|
nt = (int)ncpu < 4 ? (int)ncpu : 4;
|
||||||
else
|
else
|
||||||
nt = 1;
|
nt = 1;
|
||||||
if (ncpu > 0 && nt > ncpu)
|
if (nt > MAX_THREADS)
|
||||||
nt = (int)ncpu;
|
nt = MAX_THREADS;
|
||||||
return nt;
|
return nt;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1103,13 +1107,16 @@ static int check_sliced(void)
|
|||||||
static unsigned char buf[9001];
|
static unsigned char buf[9001];
|
||||||
int fails = 0;
|
int fails = 0;
|
||||||
size_t n, k;
|
size_t n, k;
|
||||||
|
static const int tcs[] = {
|
||||||
|
1, 2, 3, 4, 8, 12, 16, MAX_THREADS, MAX_THREADS + 1};
|
||||||
|
|
||||||
for (n = 0; n <= 9000; n += (n < 300 ? 1 : 37))
|
for (n = 0; n <= 9000; n += (n < 300 ? 1 : 37))
|
||||||
{
|
{
|
||||||
for (k = 0; k < n; k++)
|
for (k = 0; k < n; k++)
|
||||||
buf[k] = (unsigned char)rng32();
|
buf[k] = (unsigned char)rng32();
|
||||||
for (int nt = 1; nt <= 8; nt++)
|
for (size_t ti = 0; ti < sizeof tcs / sizeof tcs[0]; ti++)
|
||||||
{
|
{
|
||||||
|
int nt = tcs[ti];
|
||||||
long long tl = 0, tw = 0;
|
long long tl = 0, tw = 0;
|
||||||
long long want_l = count_newlines(buf, n);
|
long long want_l = count_newlines(buf, n);
|
||||||
int pw = 1;
|
int pw = 1;
|
||||||
@@ -1133,8 +1140,9 @@ static int check_sliced(void)
|
|||||||
{
|
{
|
||||||
for (k = 0; k < n; k++)
|
for (k = 0; k < n; k++)
|
||||||
buf[k] = (unsigned char)" \t\n\v\f\r\xa0x"[rng32() % 8];
|
buf[k] = (unsigned char)" \t\n\v\f\r\xa0x"[rng32() % 8];
|
||||||
for (int nt = 1; nt <= 8; nt++)
|
for (size_t ti = 0; ti < sizeof tcs / sizeof tcs[0]; ti++)
|
||||||
{
|
{
|
||||||
|
int nt = tcs[ti];
|
||||||
long long tl = 0, tw = 0;
|
long long tl = 0, tw = 0;
|
||||||
long long want_l = count_newlines(buf, n);
|
long long want_l = count_newlines(buf, n);
|
||||||
int pw = 1;
|
int pw = 1;
|
||||||
|
|||||||
Reference in New Issue
Block a user