diff --git a/README.md b/README.md index fa213f1..23d0446 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,22 @@ stopped being a challenge and started being a participation trophy. ## What it does ``` -fastwc [-lwc] [-m] [file...] +fastwc [OPTION]... [FILE]... +fastwc [OPTION]... --files0-from=F ``` -- `-l` lines, `-w` words, `-c` bytes, `-m` characters (multibyte) +- `-l` lines, `-w` words, `-c` bytes, `-m` characters (multibyte), + `-L` maximum line length (display width, tab stops every 8) +- `--files0-from=F` NUL-terminated file lists from F (or stdin when F is `-`) +- `--total=auto|always|only|never` control over the total row +- `--debug` report which line-counting acceleration is in use - stdin, `-`, multiple files, `total` rows, GNU-compatible counts +- A CLI surface modeled on GNU coreutils `wc` 9.11: same options, same + column alignment rule (counts are right-justified to the widest sum + of regular file sizes), same multibyte word semantics, same error + text. Swap it in and scripts keep working. +- POSIXLY_CORRECT is honored: the no-break space stops being a word + separator, exactly like GNU wc. - no `--help` in fourteen languages. One `--help`, in English, the language of people who ship software diff --git a/src/main.c b/src/main.c index cb436a8..10a221d 100644 --- a/src/main.c +++ b/src/main.c @@ -5,13 +5,22 @@ * derives the newline mask and the whitespace mask from one load, so * lines and words share a single pass over the buffer. AVX-512 when the * CPU has it, then AVX-2, then SSE2, then the scalar SWAR reference. + * + * Drop-in surface: the CLI and output are modeled on GNU coreutils wc + * 9.11 (the benchmark oracle) - option parsing via getopt_long, the + * column width rule (digits of the sum of regular file sizes, minimum + * 7 if any input is not a regular file), the total-row policy, the + * multibyte word/line-length semantics, and the error texts. */ -#define _POSIX_C_SOURCE 200809L +#define _XOPEN_SOURCE 700 /* POSIX.1-2008 XSI: wcwidth, mbrtowc, mmap, ... */ #include #include #include +#include +#include +#include #include #include #include @@ -30,10 +39,11 @@ enum { - F_LINES = 1 << 0, /* -l: count '\n' */ - F_WORDS = 1 << 1, /* -w: whitespace-separated tokens */ - F_CHARS = 1 << 2, /* -m: multibyte characters */ - F_BYTES = 1 << 3, /* -c: bytes */ + F_LINES = 1 << 0, /* -l: count '\n' */ + F_WORDS = 1 << 1, /* -w: whitespace-separated tokens */ + F_CHARS = 1 << 2, /* -m: multibyte characters */ + F_BYTES = 1 << 3, /* -c: bytes */ + F_MAXLEN = 1 << 4, /* -L: maximum display width of a line */ }; enum @@ -41,7 +51,46 @@ enum MAX_THREADS = 24, /* pick_threads() and count_sliced() agree on this */ }; +/* Long options with no short form; CHAR_MAX + 1 is the getopt convention. */ +enum +{ + OPT_HELP = CHAR_MAX + 1, + OPT_VERSION, + OPT_DEBUG, + OPT_FILES0, + OPT_TOTAL, +}; + +enum total_mode +{ + TOTAL_AUTO = 0, /* default: total row only for multiple inputs */ + TOTAL_ALWAYS, /* --total=always */ + TOTAL_ONLY, /* --total=only */ + TOTAL_NEVER, /* --total=never */ +}; + +static const char *const total_args[] = {"auto", "always", "only", "never", + NULL}; + static int flags = 0; +static int debug = 0; +static int posixly_correct = 0; +static enum total_mode total_mode = TOTAL_AUTO; +static const char *files_from = NULL; /* --files0-from=F, or NULL */ + +/* gnulib quote() style: locale quotation marks in a UTF-8 locale, + * ASCII apostrophes elsewhere. Only argmatch diagnostics use it. */ +static int curly_quotes; + +static void init_quote_style(void) +{ + const char *cs = nl_langinfo(CODESET); + + curly_quotes = cs != NULL && strstr(cs, "UTF-8") != NULL; +} + +/* Single column width shared by every count, exactly as GNU computes it. */ +static int number_width; typedef struct { @@ -49,35 +98,139 @@ typedef struct long long words; long long chars; long long bytes; - int ok; /* read succeeded */ + long long maxlen; /* -L */ + int ok; /* read succeeded */ } counts_t; +/* fstat/stat results used to derive number_width before counting. */ +typedef struct +{ + int failed; /* stat() return value; 1 = deliberately not stat'd */ + struct stat st; +} fstatus_t; + static unsigned char ws_tab[256]; /* ws_tab[c] = 1 if c separates words */ +static unsigned char + isprint_tab[256]; /* isprint_tab[c] = 1 if c is printable */ + +/* Whether byte 0xa0 separates words (locale + POSIXLY_CORRECT dependent). + * The SIMD kernels gate their NBSP compare on this so their whitespace + * expression matches ws_tab. */ +static int nbsp_ws; /* - * Word separators match GNU wc (the benchmark oracle): the six C-locale - * isspace() bytes plus U+00A0 (non-breaking space), which GNU counts - * unless POSIXLY_CORRECT is set. + * Word separators for the byte path match GNU wc 9.11: the locale's + * isspace() bytes plus U+00A0, which GNU reaches through btoc32's + * identity mapping and treats as a separator unless POSIXLY_CORRECT is + * set. In a multibyte locale this table is only consulted for single + * bytes, which is why 0xa0 stays a separator here while the multibyte + * path decides U+00A0 through is_wspace. */ static void init_ws_tab(void) { for (int i = 0; i < 256; i++) - ws_tab[i] = (isspace((unsigned char)i) || i == 0xa0) ? 1 : 0; + ws_tab[i] = + (isspace((unsigned char)i) || (i == 0xa0 && !posixly_correct)) ? 1 + : 0; + nbsp_ws = ws_tab[0xa0]; +} + +static void init_isprint_tab(void) +{ + for (int i = 0; i < 256; i++) + isprint_tab[i] = isprint((unsigned char)i) ? 1 : 0; +} + +static void try_help(void) +{ + fprintf(stderr, "Try 'fastwc --help' for more information.\n"); +} + +/* quoteaf()/quotef() style: ASCII apostrophes, shell-style $'...' for + * newlines, '\'' for embedded quotes. */ +static void qaf(FILE *out, const char *s) +{ + fputc('\'', out); + for (const char *p = s; *p; p++) + { + if (*p == '\n') + fputs("'$'\\n''", out); /* close, escape, reopen */ + else if (*p == '\'') + fputs("'\\''", out); + else + fputc(*p, out); + } + fputc('\'', out); +} + +/* quotef(): quote a file name only when it carries a shell-special or + * control byte; plain names print bare, exactly like gnulib quotef. */ +static void qfile(FILE *out, const char *s) +{ + int need = 0; + + for (size_t i = 0; s[i] != '\0'; i++) + { + unsigned char c = (unsigned char)s[i]; + + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '.' || c == '/' || + c == '-')) + { + need = 1; + break; + } + } + if (need) + qaf(out, s); + else + fputs(s, out); +} + +/* quote(): always-quoting locale style used by argmatch diagnostics. */ +static void qarg(FILE *out, const char *s) +{ + fputs(curly_quotes ? "\xe2\x80\x98" : "'", out); + fputs(s, out); + fputs(curly_quotes ? "\xe2\x80\x99" : "'", out); } static void usage(FILE *out) { - fprintf(out, "usage: fastwc [-lwc] [-m] [file...]\n" - "\n" - "Count lines, words, and bytes (default) or selected counts.\n" - "With no file, or when file is -, read standard input.\n" - "\n" - " -l count lines\n" - " -w count words\n" - " -c count bytes\n" - " -m count characters\n" - " --help display this help and exit\n" - " --version output version information and exit\n"); + fprintf( + out, + "Usage: fastwc [OPTION]... [FILE]...\n" + " or: fastwc [OPTION]... --files0-from=F\n" + "\n" + "Print newline, word, and byte counts for each FILE, and a total line " + "if\n" + "more than one FILE is specified. A word is a nonempty sequence of " + "non\n" + "white space delimited by white space characters or by start or end " + "of\n" + "input.\n" + "\n" + "With no FILE, or when FILE is -, read standard input.\n" + "\n" + "The options below may be used to select which counts are printed, " + "always\n" + "in the following order: newline, word, character, byte, maximum line\n" + "length.\n" + "\n" + " -c, --bytes print the byte counts\n" + " -m, --chars print the character counts\n" + " -l, --lines print the newline counts\n" + " --debug indicate what line count acceleration is " + "used\n" + " --files0-from=F read input from the files specified by\n" + " NUL-terminated names in file F;\n" + " If F is -, read names from standard input\n" + " -L, --max-line-length print the maximum display width\n" + " -w, --words print the word counts\n" + " --total=WHEN when to print a line with total counts;\n" + " WHEN can be: auto, always, only, never\n" + " --help display this help and exit\n" + " --version output version information and exit\n"); } /* @@ -172,9 +325,10 @@ typedef lw_t (*count_lw_fn)(const unsigned char *s, size_t n, int *prev_ws, int need_lines, int need_words); /* - * Word separators match GNU wc (the benchmark oracle): the six C-locale - * isspace() bytes plus U+00A0 (non-breaking space). The 0x09..0x0d range - * is one unsigned compare, (x - 9) < 5, plus equalities for ' ' and NBSP. + * Word separators match GNU wc (the benchmark oracle): the locale's + * isspace() bytes plus U+00A0 when it is a separator in this locale (see + * init_ws_tab). The 0x09..0x0d range is one unsigned compare, (x - 9) < 5, + * plus equalities for ' ' and NBSP. */ #if defined(__x86_64__) || defined(__i386__) @@ -209,8 +363,9 @@ count_lw_avx512(const unsigned char *s, size_t n, int *prev_ws, int need_lines, __m512i d = _mm512_sub_epi8(v, lo); uint64_t ws = nl_mask | (uint64_t)_mm512_cmpeq_epi8_mask(v, sp) | - (uint64_t)_mm512_cmpeq_epi8_mask(v, nb) | (uint64_t)_mm512_cmpeq_epi8_mask(_mm512_min_epu8(d, four), d); + if (nbsp_ws) /* 0xa0 is a separator only in some locales */ + ws |= (uint64_t)_mm512_cmpeq_epi8_mask(v, nb); words += (long long)_mm_popcnt_u64(~ws & ((ws << 1) | prev)); prev = (ws >> 63) & 1; } @@ -266,9 +421,10 @@ count_lw_avx2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, uint32_t ws = nl_mask | (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, sp)) | - (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nb)) | (uint32_t)_mm256_movemask_epi8( _mm256_cmpeq_epi8(_mm256_min_epu8(d, four), d)); + if (nbsp_ws) /* 0xa0 is a separator only in some locales */ + ws |= (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, nb)); words += (long long)_mm_popcnt_u32(~ws & ((ws << 1) | prev)); prev = (ws >> 31) & 1; } @@ -321,9 +477,10 @@ count_lw_sse2(const unsigned char *s, size_t n, int *prev_ws, int need_lines, __m128i d = _mm_sub_epi8(v, lo); uint32_t ws = nl_mask | (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, sp)) | - (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nb)) | (uint32_t)_mm_movemask_epi8( _mm_cmpeq_epi8(_mm_min_epu8(d, four), d)); + if (nbsp_ws) /* 0xa0 is a separator only in some locales */ + ws |= (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, nb)); words += (long long)popcount16(~ws & ((ws << 1) | prev)); prev = (ws >> 15) & 1; } @@ -384,14 +541,15 @@ static count_lw_fn pick_kernel(void) #endif /* - * Word separator exactly as GNU wc defines it: the Unicode white space - * set. glibc's iswspace covers it incompletely (U+2007 and U+202F are - * missing), which is why the table is explicit. + * The Unicode white space set as GNU wc 9.11 sees it: glibc's iswspace() + * (which omits the no-break spaces U+00A0, U+2007, U+202F) plus those + * three no-break spaces when POSIXLY_CORRECT is unset. The table is + * explicit because glibc's iswspace covers the set incompletely. */ static int is_wspace(wchar_t wc) { - if (wc >= 0x2000 && wc <= 0x200a) /* en space .. hair space */ - return 1; + if (wc >= 0x2000 && wc <= 0x200a) /* en space .. hair space */ + return wc != 0x2007 || !posixly_correct; /* figure space is no-break */ switch (wc) { case 0x09: /* tab */ @@ -400,43 +558,72 @@ static int is_wspace(wchar_t wc) case 0x0c: /* form feed */ case 0x0d: /* carriage return */ case 0x20: /* space */ - case 0xa0: /* no-break space */ case 0x1680: /* ogham space mark */ case 0x2028: /* line separator */ case 0x2029: /* paragraph separator */ - case 0x202f: /* narrow no-break space */ case 0x205f: /* medium mathematical space */ case 0x3000: /* ideographic space */ return 1; + case 0xa0: /* no-break space */ + case 0x202f: /* narrow no-break space */ + return !posixly_correct; default: return 0; } } /* - * Multibyte (-m) path: decode each character with mbrtowc, carrying - * incomplete sequences across read boundaries. Only used when -m is - * requested, so it stays deliberately simple. + * Maximum display width (-L) for the single-byte path, GNU-exact: a line + * ends at '\n', '\r' or '\f'; a tab advances to the next multiple of 8; + * printable bytes add one; everything else (including '\v') adds + * nothing. *linepos carries the current line across buffer boundaries. + */ +static void maxlen_single(const unsigned char *p, size_t n, long long *linepos, + long long *maxlen) +{ + for (size_t i = 0; i < n; i++) + { + unsigned char c = p[i]; + + if (c == '\t') + *linepos += 8 - (*linepos % 8); + else if (c == '\r' || c == '\f' || c == '\n') + { + if (*linepos > *maxlen) + *maxlen = *linepos; + *linepos = 0; + } + else if (c == ' ') + (*linepos)++; + else + *linepos += isprint_tab[c]; + } +} + +/* + * Multibyte path (-m, or -w/-L under a multibyte locale): decode each + * character with mbrtowc, carrying incomplete sequences across read + * boundaries. The structure is GNU wc's, so counts agree byte for byte + * with the oracle - its boundary artifacts included. */ static void count_stream_mb(FILE *fp, counts_t *c) { - /* - * GNU wc's read size and pointer accounting are replicated exactly - * (256 KiB buffer, p + prev decode pointer, p += charbytes), so - * counts agree byte for byte with the oracle - its boundary - * artifacts included. Only used when -m is requested. - */ static unsigned char buf[1 << 18]; mbstate_t st; size_t pend = 0; int in_shift = 0; - int prev_ws = 1; + int in_word = 0; + long long linepos = 0, maxlen = 0; + int need_lines = (flags & F_LINES) != 0; + int need_words = (flags & F_WORDS) != 0; + int need_chars = (flags & F_CHARS) != 0; + int need_maxlen = (flags & F_MAXLEN) != 0; memset(&st, 0, sizeof st); for (;;) { - size_t nread = fread(buf + pend, 1, sizeof buf - pend, fp); + size_t nread = fread(buf + pend, 1, sizeof buf - pend, fp); /* NOLINT */ if (nread == 0 && pend == 0) break; c->bytes += (long long)nread; @@ -450,11 +637,13 @@ static void count_stream_mb(FILE *fp, counts_t *c) { wchar_t wc; size_t charbytes; + int single_byte; if (!in_shift && *p < 0x80) { charbytes = 1; wc = *p; + single_byte = 1; } else { @@ -472,110 +661,221 @@ static void count_stream_mb(FILE *fp, counts_t *c) in_shift = 1; break; } - /* encoding error: a byte but not a character */ + /* encoding error: a byte but not a character. GNU + * treats it as a non-white-space word character. */ p++; memset(&st, 0, sizeof st); in_shift = 0; - if (prev_ws) - { + if (need_words && !in_word) c->words++; - prev_ws = 0; - } + in_word = 1; continue; } if (wc > 0x10ffff || (wc >= 0xd800 && wc <= 0xdfff)) { /* out of range: gnulib rejects, glibc does not */ p++; - if (prev_ws) - { + if (need_words && !in_word) c->words++; - prev_ws = 0; - } + in_word = 1; continue; } charbytes = r + (r == 0); + single_byte = charbytes == !in_shift; in_shift = !mbsinit(&st); } if (wc == L'\n') - c->lines++; - if (is_wspace(wc)) { - prev_ws = 1; + if (need_lines) + c->lines++; + if (need_maxlen) + { + if (linepos > maxlen) + maxlen = linepos; + linepos = 0; + } + in_word = 0; } - else if (prev_ws) + else if (wc == L'\r' || wc == L'\f') { - c->words++; - prev_ws = 0; + if (need_maxlen) + { + if (linepos > maxlen) + maxlen = linepos; + linepos = 0; + } + in_word = 0; } - c->chars++; + else if (wc == L'\t') + { + if (need_maxlen) + linepos += 8 - (linepos % 8); + in_word = 0; + } + else if (wc == L' ') + { + if (need_maxlen) + linepos++; + in_word = 0; + } + else if (wc == L'\v') + { + in_word = 0; + } + else + { + int in_word2; + + if (single_byte) + { + if (need_maxlen) + linepos += isprint_tab[(unsigned char)wc]; + in_word2 = !ws_tab[(unsigned char)wc]; + } + else + { + if (need_maxlen) + { + int w = wcwidth(wc); + if (w > 0) + linepos += w; + } + in_word2 = !is_wspace(wc); + } + if (need_words && !in_word && in_word2) + c->words++; + in_word = in_word2; + } + + if (need_chars) + c->chars++; p += charbytes; } while (p < plim); } + if (need_maxlen) + { + if (linepos > maxlen) + maxlen = linepos; + c->maxlen = maxlen; + } if (ferror(fp)) c->ok = 0; } +/* + * True when lines are counted by the SIMD byte path, which is when GNU + * runs its accelerated line counter and prints --debug diagnostics. + */ +static int using_wc_lines(void) +{ + if (flags & (F_WORDS | F_MAXLEN)) + return 0; + if ((flags & F_CHARS) && MB_CUR_MAX > 1) + return 0; + if (!(flags & F_LINES) && + ((flags & F_BYTES) || (MB_CUR_MAX <= 1 && (flags & F_CHARS)))) + return 0; + return 1; +} + +static void debug_methods(void) +{ + static int printed; + + if (printed) + return; + printed = 1; +#if defined(__x86_64__) || defined(__i386__) + __builtin_cpu_init(); + if (__builtin_cpu_supports("avx512bw")) + fprintf(stderr, "fastwc: using avx512 hardware support\n"); + else + fprintf(stderr, "fastwc: avx512 support not detected\n"); + if (__builtin_cpu_supports("avx2")) + fprintf(stderr, "fastwc: using avx2 hardware support\n"); + else + fprintf(stderr, "fastwc: avx2 support not detected\n"); +#endif +} + static void count_stream(FILE *fp, counts_t *c) { static _Alignas(64) unsigned char buf[1 << 17]; /* 128 KiB, SIMD-aligned */ + struct stat st; size_t nread; int prev_ws = 1; /* start of file: as if preceded by whitespace */ + long long linepos = 0, maxlen = 0; + int fd = fileno(fp); - if (flags & F_CHARS) + /* Under a multibyte locale, -w/-m/-L need per-character decoding; + * -l and -c alone stay on the byte path, exactly like GNU wc. */ + if ((MB_CUR_MAX > 1) && (flags & (F_CHARS | F_WORDS | F_MAXLEN))) { - if (MB_CUR_MAX <= 1) - { /* single-byte locale: every byte is a character, like GNU wc */ - for (;;) - { - nread = fread(buf, 1, sizeof buf, fp); /* NOLINT */ - if (nread == 0) - break; - c->bytes += (long long)nread; - c->chars += (long long)nread; - if (flags & (F_LINES | F_WORDS)) - { - lw_t r = - count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0, - (flags & F_WORDS) != 0); - if (flags & F_LINES) - c->lines += r.lines; - if (flags & F_WORDS) - c->words += r.words; - } - } - } - else - { - count_stream_mb(fp, c); - } - if (ferror(fp)) - c->ok = 0; + count_stream_mb(fp, c); return; } - /* Regular file — named or a stdin redirect — map instead of - * streaming: no copy, and the count can be split across cores. */ - struct stat st; - if (fstat(fileno(fp), &st) == 0 && S_ISREG(st.st_mode) && st.st_size > 0) + if (debug && using_wc_lines()) + debug_methods(); + + if (flags == F_BYTES) { - if (flags == F_BYTES) - { /* GNU wc does not read the file either */ - c->bytes = (long long)st.st_size; - return; - } - void *m = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, - fileno(fp), 0); - if (m != MAP_FAILED) + /* -c alone: GNU sizes regular files with lseek instead of + * reading, but a page-aligned st_size (proc-like files report + * an approximate size) gets a tail read. */ + static long page; + off_t end, cur, hi; + + if (page <= 0) + page = sysconf(_SC_PAGESIZE); + if (page <= 0) + page = 4096; + if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) && st.st_size >= 0) { - count_mapped((const unsigned char *)m, (size_t)st.st_size, c); - munmap(m, (size_t)st.st_size); - if (ferror(fp)) - c->ok = 0; - return; + cur = lseek(fd, 0, SEEK_CUR); + if (cur >= 0) + { + end = st.st_size; + if (end % page != 0) + { + c->bytes = end > cur ? end - cur : 0; + if (c->bytes && lseek(fd, c->bytes, SEEK_CUR) >= 0) + { + if (ferror(fp)) + c->ok = 0; + return; /* positioned at EOF; nothing left to read */ + } + c->bytes = 0; + } + else + { + hi = end - end % (st.st_blksize + 1); + if (cur < hi && lseek(fd, hi, SEEK_CUR) >= 0) + c->bytes = hi - cur; + } + } + } + /* fall through and read the tail (or everything) */ + } + else + { + /* Regular file — named or a stdin redirect — map instead of + * streaming: no copy, and the count can be split across cores. */ + if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) && st.st_size > 0) + { + void *m = + mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (m != MAP_FAILED) + { + count_mapped((const unsigned char *)m, (size_t)st.st_size, c); + munmap(m, (size_t)st.st_size); + if (ferror(fp)) + c->ok = 0; + return; + } } } @@ -585,6 +885,8 @@ static void count_stream(FILE *fp, counts_t *c) if (nread == 0) break; c->bytes += (long long)nread; + if (flags & F_CHARS) /* single-byte locale: chars == bytes */ + c->chars += (long long)nread; if (flags & (F_LINES | F_WORDS)) { lw_t r = count_lw(buf, nread, &prev_ws, (flags & F_LINES) != 0, @@ -594,6 +896,15 @@ static void count_stream(FILE *fp, counts_t *c) if (flags & F_WORDS) c->words += r.words; } + if (flags & F_MAXLEN) + maxlen_single(buf, nread, &linepos, &maxlen); + } + + if (flags & F_MAXLEN) + { + if (linepos > maxlen) + maxlen = linepos; + c->maxlen = maxlen; } if (ferror(fp)) @@ -703,6 +1014,8 @@ static void count_mapped(const unsigned char *p, size_t n, counts_t *c) long long lines = 0, words = 0; c->bytes += (long long)n; + if (flags & F_CHARS) /* single-byte locale: chars == bytes */ + c->chars += (long long)n; if (nt <= 1) { @@ -721,211 +1034,632 @@ static void count_mapped(const unsigned char *p, size_t n, counts_t *c) c->lines += lines; if (flags & F_WORDS) c->words += words; + if (flags & F_MAXLEN) + { + /* Line positions are sequential state, so -L is a single + * scalar pass over the already-mapped buffer. */ + long long linepos = 0, maxlen = 0; + + maxlen_single(p, n, &linepos, &maxlen); + c->maxlen = linepos > maxlen ? linepos : maxlen; + } } static void count_file(const char *path, counts_t *c) { FILE *fp; - if (strcmp(path, "-") == 0) + if (path == NULL || strcmp(path, "-") == 0) { + const char *label = path ? "-" : "standard input"; + count_stream(stdin, c); if (ferror(stdin)) - fprintf(stderr, "fastwc: standard input: read error: %s\n", - strerror(errno)); + { + fputs("fastwc: ", stderr); + qfile(stderr, label); + fprintf(stderr, ": %s\n", strerror(errno)); + c->ok = 0; + } return; } fp = fopen(path, "rb"); if (fp == NULL) { - fprintf(stderr, "fastwc: %s: %s\n", path, strerror(errno)); + fputs("fastwc: ", stderr); + qfile(stderr, path); + fprintf(stderr, ": %s\n", strerror(errno)); c->ok = 0; return; } - /* count_stream maps regular files itself; -c alone skips reading */ - if (flags == F_BYTES) - { - struct stat st; - if (fstat(fileno(fp), &st) == 0 && S_ISREG(st.st_mode)) - { - c->bytes = (long long)st.st_size; - fclose(fp); - return; - } - } - count_stream(fp, c); if (ferror(fp)) - fprintf(stderr, "fastwc: %s: read error: %s\n", path, strerror(errno)); + { + fputs("fastwc: ", stderr); + qfile(stderr, path); + fprintf(stderr, ": %s\n", strerror(errno)); + } fclose(fp); } -static int col_width(long long v) +/* + * The fstatus array feeds compute_number_width(). The two fast exits + * mirror GNU's get_input_fstatus(): no files (streamed --files0-from) + * or a single file with a single requested count need no stat at all. + */ +static void get_input_fstatus(int nfiles, char *const *file, fstatus_t *fs) { - int w = 1; - while (v >= 10) + int single_count = flags != 0 && (flags & (flags - 1)) == 0; + + if (nfiles == 0 || (nfiles == 1 && single_count)) + fs[0].failed = 1; + else + for (int i = 0; i < nfiles; i++) + fs[i].failed = (!file[i] || strcmp(file[i], "-") == 0) + ? fstat(STDIN_FILENO, &fs[i].st) + : stat(file[i], &fs[i].st); +} + +/* + * Column width, GNU-exact: one width for every count column, equal to + * the number of digits in the sum of the regular files' sizes, and never + * below 7 when any input is not a regular file (a pipe or device may + * deliver an arbitrarily large count). A single count for a single file + * needs no alignment and keeps width 1. + */ +static int compute_number_width(int nfiles, const fstatus_t *fs) +{ + int width = 1; + + if (nfiles > 0 && fs[0].failed <= 0) { - v /= 10; - w++; + int minimum_width = 1; + long long regular_total = 0; + + for (int i = 0; i < nfiles; i++) + if (!fs[i].failed) + { + if (!S_ISREG(fs[i].st.st_mode)) + minimum_width = 7; + else if (regular_total <= + LLONG_MAX - (long long)fs[i].st.st_size) + regular_total += (long long)fs[i].st.st_size; + else + { + regular_total = LLONG_MAX; + break; + } + } + + for (; regular_total >= 10; regular_total /= 10) + width++; + if (width < minimum_width) + width = minimum_width; } - return w; + + return width; } -static void widen(int *width, long long v, int enabled) +/* One count row, in the fixed order newline, word, character, byte, + * maximum line length, with GNU's exact spacing. */ +static void write_counts(const counts_t *c, const char *file) { - int w; + const char *fmt = "%*lld"; - if (!enabled) - return; - w = col_width(v); - if (w > *width) - *width = w; + if (flags & F_LINES) + { + printf(fmt, number_width, c->lines); + fmt = " %*lld"; + } + if (flags & F_WORDS) + { + printf(fmt, number_width, c->words); + fmt = " %*lld"; + } + if (flags & F_CHARS) + { + printf(fmt, number_width, c->chars); + fmt = " %*lld"; + } + if (flags & F_BYTES) + { + printf(fmt, number_width, c->bytes); + fmt = " %*lld"; + } + if (flags & F_MAXLEN) + printf(fmt, number_width, c->maxlen); + if (file) + { + /* GNU quotes names containing a newline (shell style) so a row + * stays one line. */ + if (strchr(file, '\n')) + { + putchar(' '); + qaf(stdout, file); + } + else + printf(" %s", file); + } + putchar('\n'); } +/* Exact or unambiguous prefix match over GNU's --total arguments. */ +static int argmatch_total(const char *arg) +{ + size_t len = strlen(arg); + int match = -1; + + for (int i = 0; total_args[i]; i++) + { + if (strcmp(total_args[i], arg) == 0) + return i; + if (strncmp(total_args[i], arg, len) == 0) + { + if (match >= 0) + return -2; /* ambiguous */ + match = i; + } + } + return match; +} + +static void argmatch_fail(int problem, const char *arg) +{ + fputs("fastwc: ", stderr); + fputs(problem == -2 ? "ambiguous argument " : "invalid argument ", stderr); + qarg(stderr, arg); + fputs(" for ", stderr); + qarg(stderr, "--total"); + fputc('\n', stderr); + fputs("Valid arguments are:\n", stderr); + for (int i = 0; total_args[i]; i++) + { + fputs(" - ", stderr); + qarg(stderr, total_args[i]); + fputc('\n', stderr); + } + try_help(); + exit(1); +} + +/* Count one input; prints its row unless --total=only. Returns 0 (and + * reports) if the input must be skipped or failed. */ +static int process_name(const char *name, long record, counts_t *c) +{ + if (files_from && strcmp(files_from, "-") == 0 && strcmp(name, "-") == 0) + { + fputs("fastwc: when reading file names from standard input, no " + "file name of ", + stderr); + qaf(stderr, name); + fputs(" allowed\n", stderr); + return 0; + } + if (!name[0]) + { + if (files_from) + { + fputs("fastwc: ", stderr); + qfile(stderr, files_from); + fprintf(stderr, ":%ld: invalid zero-length file name\n", record); + } + else + fprintf(stderr, "fastwc: invalid zero-length file name\n"); + return 0; + } + + c->ok = 1; + count_file(name, c); + if (!c->ok) + return 0; + + if (total_mode != TOTAL_ONLY) + write_counts(c, name); + return 1; +} + +/* Read one NUL-terminated name from f. Returns 1 with *out set (possibly + * an empty name), 0 at EOF, -1 on read error. An unterminated final name + * is still a name, as in GNU. */ +static int read_one_name(FILE *f, char **out) +{ + size_t cap = 256, len = 0; + char *buf = malloc(cap); + int ch; + + for (;;) + { + ch = getc(f); + if (ch == EOF) + { + if (ferror(f)) + { + free(buf); + return -1; + } + break; + } + if (ch == '\0') + { + buf[len] = '\0'; + *out = buf; + return 1; + } + if (len + 1 >= cap) + { + cap *= 2; + buf = realloc(buf, cap); + if (buf == NULL) + { + fprintf(stderr, "fastwc: out of memory\n"); + exit(1); + } + } + buf[len++] = (char)ch; + } + + if (len > 0) /* EOF with an unterminated name in hand */ + { + buf[len] = '\0'; + *out = buf; + return 1; + } + free(buf); + return 0; +} + +/* Read the whole name list (fstat'd <= 10 MiB) and split it on NULs. + * Returns the token buffer (names point into it; free it with free()); + * NULL on read error. *names_out and *n_out receive the token array + * and count, which may be zero for an empty list. */ +static char *read_all_names(FILE *f, size_t size, char ***names_out, + long *n_out) +{ + size_t cap = size + 1; + char *buf = malloc(cap); + char **names; + size_t len = 0; + long n = 0; + size_t r; + + if (buf == NULL) + { + fprintf(stderr, "fastwc: out of memory\n"); + exit(1); + } + for (;;) + { + if (len == cap) /* the file outgrew its fstat size; grow, bounded */ + { + size_t ncap = cap < (size_t)32 << 20 ? cap * 2 : cap; + char *nb = ncap == cap ? NULL : realloc(buf, ncap); + + if (nb == NULL) + { + free(buf); + fprintf(stderr, "fastwc: name list too large\n"); + exit(1); + } + buf = nb; + cap = ncap; + } + r = fread(buf + len, 1, cap - len, f); /* NOLINT: EOF-state FP */ + if (r == 0) + break; + len += r; + } + if (ferror(f)) + { + free(buf); + return NULL; + } + + /* One token per NUL, plus a trailing unterminated name. */ + for (size_t i = 0; i < len; i++) + if (buf[i] == '\0') + n++; + if (len > 0 && buf[len - 1] != '\0') + n++; + + names = n ? malloc((size_t)n * sizeof *names) /* NOLINT: multi-level */ + : NULL; + if (n && names == NULL) + { + free(buf); + fprintf(stderr, "fastwc: out of memory\n"); + exit(1); + } + for (size_t i = 0, k = 0; i < len && k < (size_t)n;) + { + names[k++] = buf + i; + while (i < len && buf[i] != '\0') + i++; + if (i < len) + i++; /* past the NUL */ + } + + *names_out = names; + *n_out = n; + return buf; +} + +typedef struct +{ + long long lines, words, chars, bytes, maxlen; +} total_t; + +static const struct option longopts[] = { + {"bytes", no_argument, NULL, 'c'}, + {"chars", no_argument, NULL, 'm'}, + {"lines", no_argument, NULL, 'l'}, + {"words", no_argument, NULL, 'w'}, + {"debug", no_argument, NULL, OPT_DEBUG}, + {"files0-from", required_argument, NULL, OPT_FILES0}, + {"max-line-length", no_argument, NULL, 'L'}, + {"total", required_argument, NULL, OPT_TOTAL}, + {"help", no_argument, NULL, OPT_HELP}, + {"version", no_argument, NULL, OPT_VERSION}, + {NULL, 0, NULL, 0}}; + #ifndef FASTWC_SELFTEST int main(int argc, char **argv) { - counts_t *rows; - int nfiles = 0; - int failed = 0; - int i, a; + int optc; + int ok = 1; + long n_args = 0; + total_t totals = {0, 0, 0, 0, 0}; + setlocale(LC_CTYPE, ""); + posixly_correct = getenv("POSIXLY_CORRECT") != NULL; + init_quote_style(); init_ws_tab(); + init_isprint_tab(); count_lw = pick_kernel(); + /* Line-buffer stdout like GNU, so parallel processes do not + * interleave their rows. */ + setvbuf(stdout, NULL, _IOLBF, 0); - for (a = 1; a < argc; a++) - { - const char *arg = argv[a]; - - if (arg[0] != '-' || arg[1] == '\0') - break; /* first file argument */ - if (strcmp(arg, "--") == 0) + while ((optc = getopt_long(argc, argv, "clLmw", longopts, NULL)) != -1) + switch (optc) { - a++; + case 'c': + flags |= F_BYTES; + break; + case 'm': + flags |= F_CHARS; + break; + case 'l': + flags |= F_LINES; + break; + case 'w': + flags |= F_WORDS; + break; + case 'L': + flags |= F_MAXLEN; + break; + case OPT_DEBUG: + debug = 1; + break; + case OPT_FILES0: + files_from = optarg; + break; + case OPT_TOTAL: + { + int m = argmatch_total(optarg); + if (m < 0) + argmatch_fail(m, optarg); + total_mode = (enum total_mode)m; break; } - if (strcmp(arg, "--help") == 0) - { + case OPT_HELP: usage(stdout); return 0; - } - if (strcmp(arg, "--version") == 0) - { + case OPT_VERSION: printf("fastwc 0.1.0\n"); return 0; + default: /* getopt_long already reported the option */ + try_help(); + return 1; } - for (const char *p = arg + 1; *p; p++) - { - switch (*p) - { - case 'l': - flags |= F_LINES; - break; - case 'w': - flags |= F_WORDS; - break; - case 'c': - flags |= F_BYTES; - break; - case 'm': - flags |= F_CHARS; - break; - default: - fprintf(stderr, "fastwc: invalid option -- '%c'\n", *p); - usage(stderr); - return 1; - } - } - } if (flags == 0) flags = F_LINES | F_WORDS | F_BYTES; /* wc default: -l -w -c */ - if (flags & F_CHARS) - setlocale(LC_CTYPE, ""); - - nfiles = argc - a; - if (nfiles == 0) + if (files_from) { - rows = calloc(1, sizeof *rows); - rows[0].ok = 1; - count_stream(stdin, &rows[0]); - if (!rows[0].ok) + FILE *stream; + struct stat st; + + /* You may not mix --files0-from with file operands. */ + if (optind < argc) { - fprintf(stderr, "fastwc: standard input: read error: %s\n", - strerror(errno)); - failed = 1; + fputs("fastwc: extra operand ", stderr); + qaf(stderr, argv[optind]); + fputc('\n', stderr); + fprintf(stderr, "file operands cannot be combined with " + "--files0-from\n"); + try_help(); + return 1; + } + + if (strcmp(files_from, "-") == 0) + stream = stdin; + else + { + stream = fopen(files_from, "r"); + if (stream == NULL) + { + fputs("fastwc: cannot open ", stderr); + qaf(stderr, files_from); + fprintf(stderr, " for reading: %s\n", strerror(errno)); + return 1; + } + } + + /* Small regular name lists are read in full so the files can be + * stat'd and the columns aligned; anything else is streamed a + * name at a time with the minimum width, like GNU. */ + if (fstat(fileno(stream), &st) == 0 && S_ISREG(st.st_mode) && + st.st_size <= (off_t)10 * 1024 * 1024) + { + long n = 0; + char **names = NULL; + char *names_buf = + read_all_names(stream, (size_t)st.st_size, &names, &n); + fclose(stream); + + if (names_buf == NULL) + { + fputs("fastwc: cannot read file names from ", stderr); + qaf(stderr, files_from); + fputc('\n', stderr); + return 1; + } + + fstatus_t *fs = calloc((size_t)(n ? n : 1), sizeof *fs); + get_input_fstatus((int)n, names, fs); + number_width = + total_mode == TOTAL_ONLY ? 1 : compute_number_width((int)n, fs); + free(fs); + + for (long i = 0; i < n; i++) + { + counts_t c; + + memset(&c, 0, sizeof c); + if (process_name(names[i], i + 1, &c)) + { + totals.lines += c.lines; + totals.words += c.words; + totals.chars += c.chars; + totals.bytes += c.bytes; + if (c.maxlen > totals.maxlen) + totals.maxlen = c.maxlen; + } + else + ok = 0; + } + n_args = n; + free(names); /* NOLINT: multi-level free is valid C */ + free(names_buf); + } + else + { + char *name; + int r; + + /* Streamed names get the unaligned width. */ + number_width = 1; + + while ((r = read_one_name(stream, &name)) > 0) + { + counts_t c; + + n_args++; + memset(&c, 0, sizeof c); + if (process_name(name, n_args, &c)) + { + totals.lines += c.lines; + totals.words += c.words; + totals.chars += c.chars; + totals.bytes += c.bytes; + if (c.maxlen > totals.maxlen) + totals.maxlen = c.maxlen; + } + else + ok = 0; + free(name); + } + if (r < 0) + { + fputs("fastwc: ", stderr); + qfile(stderr, files_from); + fprintf(stderr, ": read error: %s\n", strerror(errno)); + ok = 0; + } + fclose(stream); } - nfiles = 1; } else { - rows = calloc((size_t)nfiles, sizeof *rows); - for (i = 0; i < nfiles; i++) + int nfiles = argc - optind; + fstatus_t *fs; + + if (nfiles == 0) { - rows[i].ok = 1; - count_file(argv[a + i], &rows[i]); - if (!rows[i].ok) - failed = 1; + counts_t c; + + n_args = 0; + fs = calloc(1, sizeof *fs); + get_input_fstatus(1, (char *[]){NULL}, fs); + number_width = + total_mode == TOTAL_ONLY ? 1 : compute_number_width(1, fs); + free(fs); + + memset(&c, 0, sizeof c); + c.ok = 1; + count_stream(stdin, &c); + if (!c.ok) + { + fprintf(stderr, "fastwc: standard input: %s\n", + strerror(errno)); + ok = 0; + } + else if (total_mode != TOTAL_ONLY) + write_counts(&c, NULL); + totals.lines += c.lines; + totals.words += c.words; + totals.chars += c.chars; + totals.bytes += c.bytes; + if (c.maxlen > totals.maxlen) + totals.maxlen = c.maxlen; + } + else + { + fs = calloc((size_t)nfiles, sizeof *fs); + get_input_fstatus(nfiles, argv + optind, fs); + number_width = + total_mode == TOTAL_ONLY ? 1 : compute_number_width(nfiles, fs); + free(fs); + + n_args = nfiles; + for (int i = 0; i < nfiles; i++) + { + counts_t c; + + memset(&c, 0, sizeof c); + if (process_name(argv[optind + i], i + 1, &c)) + { + totals.lines += c.lines; + totals.words += c.words; + totals.chars += c.chars; + totals.bytes += c.bytes; + if (c.maxlen > totals.maxlen) + totals.maxlen = c.maxlen; + } + else + ok = 0; + } } } - /* Column widths: widest count in each column across rows + total. */ - int wl = 1, ww = 1, wm = 1, wb = 1; - long long tl = 0, tw = 0, tm = 0, tb = 0; - - for (i = 0; i < nfiles; i++) + if (total_mode != TOTAL_NEVER && (total_mode != TOTAL_AUTO || n_args > 1)) { - counts_t *r = &rows[i]; - tl += r->lines; - tw += r->words; - tm += r->chars; - tb += r->bytes; - widen(&wl, r->lines, flags & F_LINES); - widen(&ww, r->words, flags & F_WORDS); - widen(&wm, r->chars, flags & F_CHARS); - widen(&wb, r->bytes, flags & F_BYTES); - } - widen(&wl, tl, flags & F_LINES); - widen(&ww, tw, flags & F_WORDS); - widen(&wm, tm, flags & F_CHARS); - widen(&wb, tb, flags & F_BYTES); + counts_t t; - for (i = 0; i < nfiles; i++) - { - counts_t *r = &rows[i]; - if (flags & F_LINES) - printf("%*lld ", wl, r->lines); - if (flags & F_WORDS) - printf("%*lld ", ww, r->words); - if (flags & F_CHARS) - printf("%*lld ", wm, r->chars); - if (flags & F_BYTES) - printf("%*lld ", wb, r->bytes); - if (argc - a > 0) - printf("%s", argv[a + i]); - printf("\n"); + t.lines = totals.lines; + t.words = totals.words; + t.chars = totals.chars; + t.bytes = totals.bytes; + t.maxlen = totals.maxlen; + write_counts(&t, total_mode != TOTAL_ONLY ? "total" : NULL); } - if (argc - a > 1) - { - if (flags & F_LINES) - printf("%*lld ", wl, tl); - if (flags & F_WORDS) - printf("%*lld ", ww, tw); - if (flags & F_CHARS) - printf("%*lld ", wm, tm); - if (flags & F_BYTES) - printf("%*lld ", wb, tb); - printf("total\n"); - } - - free(rows); - return failed ? 1 : 0; + return ok ? 0 : 1; } #else /* FASTWC_SELFTEST */ @@ -979,7 +1713,8 @@ static lw_t count_lw_avx512_mirror(const unsigned char *s, size_t n, if (c == '\n') nl_mask |= bit; - if (c == '\n' || c == ' ' || c == 0xa0 || (uint8_t)(c - 9) < 5) + if (c == '\n' || c == ' ' || (nbsp_ws && c == 0xa0) || + (uint8_t)(c - 9) < 5) ws |= bit; } if (need_lines) @@ -1170,6 +1905,25 @@ int main(void) int fails = 0; const char *force = getenv("FASTWC_SELFTEST_FORCE"); + /* The CLI build's helpers are compiled out of reach here. */ + (void)usage; + (void)try_help; + (void)init_isprint_tab; + (void)init_quote_style; + (void)qaf; + (void)qfile; + (void)qarg; + (void)pick_kernel; + (void)get_input_fstatus; + (void)compute_number_width; + (void)argmatch_total; + (void)argmatch_fail; + (void)process_name; + (void)read_one_name; + (void)read_all_names; + (void)longopts; + + posixly_correct = getenv("POSIXLY_CORRECT") != NULL; init_ws_tab(); if (force == NULL || strcmp(force, "scalar") == 0) fails += check_kernel("scalar", count_lw_scalar);