From 50afaa17ca2e5b1fb850f9a931871018c5cff699 Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Fri, 28 Aug 2026 20:28:57 -0400 Subject: [PATCH] feat(kdl): add KDL lexer with source spans --- src/kdl/lexer.c | 943 ++++++++++++++++++++++++++++++++++++++++ src/kdl/lexer.h | 119 +++++ tests/unit/test_lexer.c | 624 ++++++++++++++++++++++++++ 3 files changed, 1686 insertions(+) create mode 100644 src/kdl/lexer.c create mode 100644 src/kdl/lexer.h create mode 100644 tests/unit/test_lexer.c diff --git a/src/kdl/lexer.c b/src/kdl/lexer.c new file mode 100644 index 0000000..a7caf90 --- /dev/null +++ b/src/kdl/lexer.c @@ -0,0 +1,943 @@ +/* + * lexer.c - KDL 2.0.0 tokenizer for stupidtools (todo 6). + * + * Produces a FLAT token stream with a source span on every token. It does + * NOT parse node/argument/property structure (the parser does, todo 7). + * See lexer.h for the full API + error-span-lifetime contract. + * + * Design decisions (mirrored in .omo/notepads/stupidtools/learnings.md): + * - identifiers are scanned greedily; non-ASCII bytes are identifier + * characters (the non-identifier set is the ASCII one from §3.10.2). + * - bare `inf` / `-inf` / `nan` / `true` / `false` / `null` are syntax + * errors; the `#`-prefixed keyword forms are distinct token kinds. + * - multi-line strings are delimited by `"""` only; the spec's "first + * line must be a newline" + dedent-prefix rules are VALUE rules and + * are deferred to the value model (todo 8), so `"""multi"""` tokenizes. + * - columns are byte-based (not codepoints); full Unicode grapheme + * columns and disallowed-literal-code-point validation are deferred. + * + * Copyright (c) 2026 huntedbytheirs + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "kdl/lexer.h" + +#include "error.h" + +#include +#include +#include + +struct st_lexer { + const char *src; /* borrowed source buffer (NUL-terminated) */ + const char *pos; /* current scan position */ + const char *filename; /* borrowed; stamped into every span */ + size_t line; /* current 1-based line */ + size_t col; /* current 1-based column (bytes) */ + struct st_span err_span; /* stable span for the most recent error */ +}; + +/* + * Character classes. Non-ASCII bytes are deliberately treated as + * identifier characters (see header note); the classification below only + * special-cases ASCII + the UTF-8 newline sequences (NEL/LS/PS). + */ + +static bool +is_digit(unsigned char c) +{ + return c >= '0' && c <= '9'; +} + +static bool +is_hex_digit(unsigned char c) +{ + return is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +static bool +is_oct_digit(unsigned char c) +{ + return c >= '0' && c <= '7'; +} + +static bool +is_bin_digit(unsigned char c) +{ + return c == '0' || c == '1'; +} + +/* Non-newline whitespace (ASCII subset; full Unicode White_Space deferred). */ +static bool +is_space(unsigned char c) +{ + return c == ' ' || c == '\t'; +} + +/* Non-identifier ASCII chars per KDL §3.10.2: \ / ( ) { } ; [ ] " # = */ +static bool +is_nonident(unsigned char c) +{ + switch (c) { + case '\\': + case '/': + case '(': + case ')': + case '{': + case '}': + case ';': + case '[': + case ']': + case '"': + case '#': + case '=': + return true; + default: + return false; + } +} + +/* Newline sequences per KDL §3.18: CRLF (one newline), CR, LF, VT, FF, + * NEL (U+0085), LS (U+2028), PS (U+2029). */ +static bool +is_newline_at(const char *p) +{ + unsigned char c = (unsigned char)p[0]; + + if (c == '\r' || c == '\n' || c == '\v' || c == '\f') { + return true; + } + if (c == 0xC2 && (unsigned char)p[1] == 0x85) { + return true; /* NEL */ + } + if (c == 0xE2 && (unsigned char)p[1] == 0x80 && + ((unsigned char)p[2] == 0xA8 || (unsigned char)p[2] == 0xA9)) { + return true; /* LS, PS */ + } + return false; +} + +/* Advance one character, updating line/col. CRLF counts as one newline. */ +static void +advance(struct st_lexer *lx) +{ + unsigned char c = (unsigned char)*lx->pos; + + if (c == '\0') { + return; + } + if (is_newline_at(lx->pos)) { + if (c == '\r' && lx->pos[1] == '\n') { + lx->pos += 2; + } else if (c == 0xC2) { + lx->pos += 2; + } else if (c == 0xE2) { + lx->pos += 3; + } else { + lx->pos += 1; + } + lx->line++; + lx->col = 1; + return; + } + lx->pos += 1; + lx->col += 1; +} + +/* True when the cursor is on a byte that may continue an identifier. */ +static bool +at_ident_char(const struct st_lexer *lx) +{ + unsigned char c = (unsigned char)*lx->pos; + + if (c == '\0' || is_nonident(c) || is_space(c)) { + return false; + } + return !is_newline_at(lx->pos); +} + +/* True when the cursor is on a byte that may START an identifier (i.e. not + * a digit, non-ident char, whitespace, or newline). `+`/`-`/`.` are handled + * by the caller because they can also begin numbers. */ +static bool +at_ident_start(const struct st_lexer *lx) +{ + unsigned char c = (unsigned char)*lx->pos; + + if (c == '\0' || is_digit(c) || is_nonident(c) || is_space(c)) { + return false; + } + return !is_newline_at(lx->pos); +} + +/* Build a KDL-parse error at `sp` and return ST_TOK_ERROR. The span is + * stored in the lexer so the error's borrowed pointer stays valid until + * the next st_lexer_next/free (see lexer.h). */ +static enum st_token_kind +fail(struct st_lexer *lx, struct st_span sp, const char *msg, + struct st_error **err) +{ + if (err != NULL) { + lx->err_span = sp; + *err = st_error_at(st_error_kdl_parse(msg), &lx->err_span); + } + return ST_TOK_ERROR; +} + +static void +emit(struct st_token *out, enum st_token_kind kind, struct st_span span, + const char *text, size_t len) +{ + out->kind = kind; + out->span = span; + out->text = text; + out->len = len; +} + +/* Is `s` (len bytes) one of the bare keyword identifiers that KDL forbids + * as an identifier string (must be written with `#` or quoted)? */ +static bool +is_bare_keyword(const char *s, size_t len) +{ + static const char *const keywords[] = { + "true", "false", "null", "inf", "-inf", "nan", + }; + size_t i; + + for (i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) { + size_t klen = strlen(keywords[i]); + if (len == klen && memcmp(s, keywords[i], klen) == 0) { + return true; + } + } + return false; +} + +/* --- line continuations ------------------------------------------------ */ + +/* Consume an escline: '\' ws* (single-line-comment | newline | eof). + * Returns 0 on success, -1 on error (*err set). */ +static int +lex_escline(struct st_lexer *lx, struct st_error **err) +{ + struct st_span sp = { lx->filename, lx->line, lx->col }; + + advance(lx); /* '\' */ + while (*lx->pos != '\0' && is_space((unsigned char)*lx->pos)) { + advance(lx); + } + if (*lx->pos == '\0') { + return 0; /* eof is a legal escline terminator */ + } + if (is_newline_at(lx->pos)) { + advance(lx); + return 0; + } + if (lx->pos[0] == '/' && lx->pos[1] == '/') { + advance(lx); + advance(lx); + while (*lx->pos != '\0' && !is_newline_at(lx->pos)) { + advance(lx); + } + if (*lx->pos != '\0') { + advance(lx); /* the terminating newline */ + } + return 0; + } + (void)fail(lx, sp, + "invalid line continuation: '\\' must be followed by a newline " + "or a line comment", err); + return -1; +} + +/* --- identifiers ------------------------------------------------------- */ + +static enum st_token_kind +lex_ident(struct st_lexer *lx, struct st_token *out, struct st_span start, + struct st_error **err) +{ + const char *begin = lx->pos; + + advance(lx); /* first char is a valid ident-start (caller guarantees) */ + while (at_ident_char(lx)) { + advance(lx); + } + + if (is_bare_keyword(begin, (size_t)(lx->pos - begin))) { + return fail(lx, start, + "bare keyword identifier; write it as a '#'-prefixed keyword " + "or a quoted string", err); + } + + emit(out, ST_TOK_IDENT, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_IDENT; +} + +/* --- numbers ----------------------------------------------------------- */ + +static enum st_token_kind +lex_number(struct st_lexer *lx, struct st_token *out, struct st_span start, + struct st_error **err) +{ + const char *begin = lx->pos; + + if (*lx->pos == '+' || *lx->pos == '-') { + advance(lx); /* sign */ + } + + if (lx->pos[0] == '0' && (lx->pos[1] == 'x' || lx->pos[1] == 'X')) { + advance(lx); + advance(lx); /* 0x */ + if (!is_hex_digit((unsigned char)*lx->pos)) { + return fail(lx, start, + "invalid hexadecimal number: expected a hex digit after " + "'0x'", err); + } + while (is_hex_digit((unsigned char)*lx->pos) || + *lx->pos == '_') { + advance(lx); + } + } else if (lx->pos[0] == '0' && + (lx->pos[1] == 'o' || lx->pos[1] == 'O')) { + advance(lx); + advance(lx); /* 0o */ + if (!is_oct_digit((unsigned char)*lx->pos)) { + return fail(lx, start, + "invalid octal number: expected an octal digit after '0o'", + err); + } + while (is_oct_digit((unsigned char)*lx->pos) || *lx->pos == '_') { + advance(lx); + } + } else if (lx->pos[0] == '0' && + (lx->pos[1] == 'b' || lx->pos[1] == 'B')) { + advance(lx); + advance(lx); /* 0b */ + if (!is_bin_digit((unsigned char)*lx->pos)) { + return fail(lx, start, + "invalid binary number: expected a binary digit after '0b'", + err); + } + while (is_bin_digit((unsigned char)*lx->pos) || *lx->pos == '_') { + advance(lx); + } + } else { + /* decimal: integer ('.' integer)? exponent? */ + while (is_digit((unsigned char)*lx->pos) || *lx->pos == '_') { + advance(lx); + } + if (*lx->pos == '.' && is_digit((unsigned char)lx->pos[1])) { + advance(lx); /* '.' */ + while (is_digit((unsigned char)*lx->pos) || *lx->pos == '_') { + advance(lx); + } + } + if (*lx->pos == 'e' || *lx->pos == 'E') { + const char *save = lx->pos; + size_t save_line = lx->line; + size_t save_col = lx->col; + + advance(lx); /* e */ + if (*lx->pos == '+' || *lx->pos == '-') { + advance(lx); + } + if (!is_digit((unsigned char)*lx->pos)) { + /* not a real exponent: back off */ + lx->pos = save; + lx->line = save_line; + lx->col = save_col; + } else { + while (is_digit((unsigned char)*lx->pos) || + *lx->pos == '_') { + advance(lx); + } + } + } + } + + emit(out, ST_TOK_NUMBER, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_NUMBER; +} + +/* Dispatch for bytes that may start either a number or an identifier + * (digits, '+', '-', '.'). */ +static enum st_token_kind +lex_value_start(struct st_lexer *lx, struct st_token *out, + struct st_span start, struct st_error **err) +{ + unsigned char c = (unsigned char)*lx->pos; + + if (is_digit(c)) { + return lex_number(lx, out, start, err); + } + if (c == '+' || c == '-') { + unsigned char n = (unsigned char)lx->pos[1]; + + if (is_digit(n)) { + return lex_number(lx, out, start, err); + } + if (n == '.' && is_digit((unsigned char)lx->pos[2])) { + return fail(lx, start, + "number must have a leading digit before the decimal point", + err); + } + return lex_ident(lx, out, start, err); + } + if (c == '.') { + if (is_digit((unsigned char)lx->pos[1])) { + return fail(lx, start, + "number must have a leading digit (write '0.1', not '.1')", + err); + } + return lex_ident(lx, out, start, err); + } + return lex_ident(lx, out, start, err); +} + +/* --- strings ----------------------------------------------------------- */ + +/* Scan one escape sequence (at a '\'); validates it but does not unescape. + * Returns 0 on success, -1 on error. */ +static int +lex_escape(struct st_lexer *lx, struct st_span start, struct st_error **err) +{ + unsigned char c; + + advance(lx); /* '\' */ + c = (unsigned char)*lx->pos; + + if (c == '\0') { + (void)fail(lx, start, "unterminated escape sequence in string", err); + return -1; + } + switch (c) { + case 'n': + case 'r': + case 't': + case '\\': + case '"': + case 'b': + case 'f': + case 's': + advance(lx); + return 0; + case 'u': { + int ndigits = 0; + + advance(lx); /* u */ + if (*lx->pos != '{') { + (void)fail(lx, start, + "invalid unicode escape: expected '{' after '\\u'", err); + return -1; + } + advance(lx); /* { */ + while (is_hex_digit((unsigned char)*lx->pos)) { + advance(lx); + ndigits++; + } + if (*lx->pos != '}') { + (void)fail(lx, start, + "invalid unicode escape: expected '}'", err); + return -1; + } + if (ndigits == 0 || ndigits > 6) { + (void)fail(lx, start, + "invalid unicode escape: expected 1-6 hex digits", err); + return -1; + } + advance(lx); /* } */ + return 0; + } + default: + /* whitespace escape: '\' + one-or-more whitespace/newlines */ + if (is_space(c) || is_newline_at(lx->pos)) { + while (*lx->pos != '\0' && + (is_space((unsigned char)*lx->pos) || + is_newline_at(lx->pos))) { + advance(lx); + } + return 0; + } + (void)fail(lx, start, "invalid escape sequence in string", err); + return -1; + } +} + +/* Single-line quoted string: "..." (raw disables escapes). */ +static enum st_token_kind +lex_quoted_string(struct st_lexer *lx, struct st_token *out, + struct st_span start, struct st_error **err, bool raw) +{ + const char *begin = lx->pos; + + advance(lx); /* opening '"' */ + + for (;;) { + unsigned char c = (unsigned char)*lx->pos; + + if (c == '\0') { + return fail(lx, start, "unterminated string literal", err); + } + if (c == '"') { + advance(lx); /* closing '"' */ + break; + } + if (!raw && c == '\\') { + if (lex_escape(lx, start, err) < 0) { + return ST_TOK_ERROR; + } + continue; + } + if (is_newline_at(lx->pos)) { + return fail(lx, start, + "unterminated string literal (newline before closing quote)", + err); + } + advance(lx); + } + + emit(out, ST_TOK_STRING, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_STRING; +} + +/* Multi-line string: """...""" (raw disables escapes). Only the delimiters + * are recognized here; dedent/value rules are deferred to todo 8. */ +static enum st_token_kind +lex_multiline_string(struct st_lexer *lx, struct st_token *out, + struct st_span start, struct st_error **err, bool raw) +{ + const char *begin = lx->pos; + + advance(lx); + advance(lx); + advance(lx); /* opening '"""' */ + + for (;;) { + if (*lx->pos == '\0') { + return fail(lx, start, "unterminated multi-line string", err); + } + if (lx->pos[0] == '"' && lx->pos[1] == '"' && lx->pos[2] == '"') { + advance(lx); + advance(lx); + advance(lx); /* closing '"""' */ + break; + } + if (!raw && *lx->pos == '\\') { + if (lex_escape(lx, start, err) < 0) { + return ST_TOK_ERROR; + } + continue; + } + advance(lx); + } + + emit(out, ST_TOK_MULTILINE_STRING, start, begin, + (size_t)(lx->pos - begin)); + return ST_TOK_MULTILINE_STRING; +} + +/* Raw string: '#'* '"' body '"' '#'* or '#'* '"""' body '"""' '#'*. + * The body may contain '"' / '#' as long as it is not the closing + * delimiter (the closing quote followed by exactly n '#'). */ +static enum st_token_kind +lex_raw_string(struct st_lexer *lx, struct st_token *out, + struct st_span start, struct st_error **err) +{ + const char *begin = lx->pos; + size_t n = 0; + size_t i; + bool multi; + + while (*lx->pos == '#') { + advance(lx); + n++; + } + + multi = (lx->pos[0] == '"' && lx->pos[1] == '"' && lx->pos[2] == '"'); + + if (multi) { + advance(lx); + advance(lx); + advance(lx); /* '"""' */ + for (;;) { + if (*lx->pos == '\0') { + return fail(lx, start, + "unterminated raw multi-line string", err); + } + if (lx->pos[0] == '"' && lx->pos[1] == '"' && + lx->pos[2] == '"') { + bool closes = true; + + for (i = 0; i < n; i++) { + if (lx->pos[3 + i] != '#') { + closes = false; + break; + } + } + if (closes) { + advance(lx); + advance(lx); + advance(lx); + for (i = 0; i < n; i++) { + advance(lx); + } + break; + } + } + advance(lx); + } + } else { + advance(lx); /* '"' */ + for (;;) { + if (*lx->pos == '\0') { + return fail(lx, start, + "unterminated raw string literal", err); + } + if (lx->pos[0] == '"') { + bool closes = true; + + for (i = 0; i < n; i++) { + if (lx->pos[1 + i] != '#') { + closes = false; + break; + } + } + if (closes) { + advance(lx); /* '"' */ + for (i = 0; i < n; i++) { + advance(lx); + } + break; + } + } + if (is_newline_at(lx->pos)) { + return fail(lx, start, + "unterminated raw string literal (newline before " + "closing quote)", err); + } + advance(lx); + } + } + + emit(out, ST_TOK_RAW_STRING, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_RAW_STRING; +} + +/* --- keywords (#inf / #-inf / #nan / #true / #false / #null) ----------- */ + +static enum st_token_kind +lex_keyword(struct st_lexer *lx, struct st_token *out, struct st_span start, + struct st_error **err) +{ + const char *begin = lx->pos; + const char *kw; + size_t kwlen; + + advance(lx); /* '#' */ + kw = lx->pos; + if (*lx->pos == '-') { + advance(lx); + } + while (at_ident_char(lx)) { + advance(lx); + } + kwlen = (size_t)(lx->pos - kw); + + if (kwlen == 3 && memcmp(kw, "inf", 3) == 0) { + emit(out, ST_TOK_INF, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_INF; + } + if (kwlen == 4 && memcmp(kw, "-inf", 4) == 0) { + emit(out, ST_TOK_NEG_INF, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_NEG_INF; + } + if (kwlen == 3 && memcmp(kw, "nan", 3) == 0) { + emit(out, ST_TOK_NAN, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_NAN; + } + if (kwlen == 4 && memcmp(kw, "true", 4) == 0) { + emit(out, ST_TOK_TRUE, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_TRUE; + } + if (kwlen == 5 && memcmp(kw, "false", 5) == 0) { + emit(out, ST_TOK_FALSE, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_FALSE; + } + if (kwlen == 4 && memcmp(kw, "null", 4) == 0) { + emit(out, ST_TOK_NULL, start, begin, (size_t)(lx->pos - begin)); + return ST_TOK_NULL; + } + return fail(lx, start, "unknown '#'-prefixed keyword", err); +} + +/* Dispatch on '#': raw string or keyword. */ +static enum st_token_kind +lex_hash(struct st_lexer *lx, struct st_token *out, struct st_span start, + struct st_error **err) +{ + const char *p = lx->pos; + + while (*p == '#') { + p++; + } + if (*p == '"') { + return lex_raw_string(lx, out, start, err); + } + if (lx->pos[1] == '#') { + return fail(lx, start, + "invalid token: '#' must be followed by a string or a keyword", + err); + } + return lex_keyword(lx, out, start, err); +} + +/* --- type annotation --------------------------------------------------- */ + +static enum st_token_kind +lex_type(struct st_lexer *lx, struct st_token *out, struct st_span start, + struct st_error **err) +{ + const char *name; + size_t namelen; + + advance(lx); /* '(' */ + while (is_space((unsigned char)*lx->pos)) { + advance(lx); + } + if (!at_ident_start(lx)) { + return fail(lx, start, "expected a type name inside '('", err); + } + name = lx->pos; + advance(lx); + while (at_ident_char(lx)) { + advance(lx); + } + namelen = (size_t)(lx->pos - name); + + while (is_space((unsigned char)*lx->pos)) { + advance(lx); + } + if (*lx->pos != ')') { + return fail(lx, start, + "unterminated type annotation (expected ')')", err); + } + advance(lx); /* ')' */ + + emit(out, ST_TOK_TYPE, start, name, namelen); + return ST_TOK_TYPE; +} + +/* --- comments ---------------------------------------------------------- */ + +static enum st_token_kind +lex_comment(struct st_lexer *lx, struct st_token *out, struct st_span start, + struct st_error **err) +{ + const char *begin = lx->pos; + + if (lx->pos[1] == '/') { + advance(lx); + advance(lx); /* '//' */ + while (*lx->pos != '\0' && !is_newline_at(lx->pos)) { + advance(lx); + } + emit(out, ST_TOK_LINE_COMMENT, start, begin, + (size_t)(lx->pos - begin)); + return ST_TOK_LINE_COMMENT; + } + if (lx->pos[1] == '*') { + int depth = 1; + + advance(lx); + advance(lx); /* slash-star opener */ + while (depth > 0) { + if (*lx->pos == '\0') { + return fail(lx, start, "unterminated block comment", err); + } + if (lx->pos[0] == '/' && lx->pos[1] == '*') { + advance(lx); + advance(lx); + depth++; + continue; + } + if (lx->pos[0] == '*' && lx->pos[1] == '/') { + advance(lx); + advance(lx); + depth--; + continue; + } + advance(lx); + } + emit(out, ST_TOK_BLOCK_COMMENT, start, begin, + (size_t)(lx->pos - begin)); + return ST_TOK_BLOCK_COMMENT; + } + if (lx->pos[1] == '-') { + advance(lx); + advance(lx); /* '/-' */ + emit(out, ST_TOK_SLASHDASH, start, begin, + (size_t)(lx->pos - begin)); + return ST_TOK_SLASHDASH; + } + return fail(lx, start, "unexpected '/'", err); +} + +/* --- public API -------------------------------------------------------- */ + +struct st_lexer * +st_lexer_new(const char *src, const char *filename) +{ + struct st_lexer *lx = calloc(1, sizeof(*lx)); + + if (lx == NULL) { + return NULL; + } + lx->src = src != NULL ? src : ""; + lx->pos = lx->src; + lx->filename = filename; + lx->line = 1; + lx->col = 1; + + /* Skip a leading BOM (U+FEFF = EF BB BF), legal only as the very first + * code point of a document. */ + if ((unsigned char)lx->pos[0] == 0xEF && + (unsigned char)lx->pos[1] == 0xBB && + (unsigned char)lx->pos[2] == 0xBF) { + lx->pos += 3; + } + return lx; +} + +void +st_lexer_free(struct st_lexer *lx) +{ + free(lx); +} + +enum st_token_kind +st_lexer_next(struct st_lexer *lx, struct st_token *out, struct st_error **err) +{ + unsigned char c; + struct st_span start; + + if (err != NULL) { + *err = NULL; + } + + /* Skip whitespace, newlines, and line continuations. */ + for (;;) { + while (*lx->pos != '\0' && + (is_space((unsigned char)*lx->pos) || is_newline_at(lx->pos))) { + advance(lx); + } + if (*lx->pos == '\\') { + if (lex_escline(lx, err) < 0) { + return ST_TOK_ERROR; + } + continue; + } + break; + } + + if (*lx->pos == '\0') { + emit(out, ST_TOK_EOF, (struct st_span){ lx->filename, lx->line, + lx->col }, lx->pos, 0); + return ST_TOK_EOF; + } + + start = (struct st_span){ lx->filename, lx->line, lx->col }; + c = (unsigned char)*lx->pos; + + /* numbers, signs, and dotted identifiers */ + if (is_digit(c) || c == '+' || c == '-' || c == '.') { + return lex_value_start(lx, out, start, err); + } + + switch (c) { + case '{': + emit(out, ST_TOK_LBRACE, start, lx->pos, 1); + advance(lx); + break; + case '}': + emit(out, ST_TOK_RBRACE, start, lx->pos, 1); + advance(lx); + break; + case ';': + emit(out, ST_TOK_SEMICOLON, start, lx->pos, 1); + advance(lx); + break; + case '=': + emit(out, ST_TOK_EQUALS, start, lx->pos, 1); + advance(lx); + break; + case '"': + if (lx->pos[1] == '"' && lx->pos[2] == '"') { + return lex_multiline_string(lx, out, start, err, false); + } + return lex_quoted_string(lx, out, start, err, false); + case '#': + return lex_hash(lx, out, start, err); + case '(': + return lex_type(lx, out, start, err); + case '/': + return lex_comment(lx, out, start, err); + default: + if (at_ident_start(lx)) { + return lex_ident(lx, out, start, err); + } + return fail(lx, start, "unexpected character in source", err); + } + + return out->kind; +} + +const char * +st_token_kind_name(enum st_token_kind kind) +{ + switch (kind) { + case ST_TOK_EOF: + return "eof"; + case ST_TOK_ERROR: + return "error"; + case ST_TOK_LBRACE: + return "lbrace"; + case ST_TOK_RBRACE: + return "rbrace"; + case ST_TOK_SEMICOLON: + return "semicolon"; + case ST_TOK_EQUALS: + return "equals"; + case ST_TOK_IDENT: + return "ident"; + case ST_TOK_STRING: + return "string"; + case ST_TOK_MULTILINE_STRING: + return "multiline-string"; + case ST_TOK_RAW_STRING: + return "raw-string"; + case ST_TOK_NUMBER: + return "number"; + case ST_TOK_INF: + return "inf"; + case ST_TOK_NEG_INF: + return "-inf"; + case ST_TOK_NAN: + return "nan"; + case ST_TOK_TRUE: + return "true"; + case ST_TOK_FALSE: + return "false"; + case ST_TOK_NULL: + return "null"; + case ST_TOK_TYPE: + return "type"; + case ST_TOK_LINE_COMMENT: + return "line-comment"; + case ST_TOK_BLOCK_COMMENT: + return "block-comment"; + case ST_TOK_SLASHDASH: + return "slashdash"; + default: + return "?"; + } +} diff --git a/src/kdl/lexer.h b/src/kdl/lexer.h new file mode 100644 index 0000000..c8abf27 --- /dev/null +++ b/src/kdl/lexer.h @@ -0,0 +1,119 @@ +/* + * lexer.h - KDL 2.0.0 tokenizer for stupidtools. + * + * Produces a FLAT token stream; it does NOT parse node/argument/property + * structure (that is the parser, todo 7). Every token carries a source + * span (struct st_span) and a borrowed text slice into the source buffer. + * + * API + * --- + * struct st_lexer *st_lexer_new(const char *src, const char *filename); + * enum st_token_kind st_lexer_next(struct st_lexer *, struct st_token *, + * struct st_error **err); + * void st_lexer_free(struct st_lexer *); + * const char *st_token_kind_name(enum st_token_kind); + * + * st_lexer_new() borrows `src` (never copies); the buffer must remain + * valid for the lexer's lifetime because token text slices point into it. + * `filename` is also borrowed and is stamped into every span. + * + * st_lexer_next() returns one token per call. It returns: + * - a token kind (>= ST_TOK_LBRACE) and fills `*out` on success, + * - ST_TOK_EOF at end of input (out->text/len are empty), + * - ST_TOK_ERROR on a lex error, in which case `*err` is set to a + * freshly allocated st_error (category ST_ERR_KDL_PARSE) and `*out` + * is left unchanged. The caller owns `*err` and frees it with + * st_error_free(). + * + * ERROR-SPAN LIFETIME (borrowed span) + * --- + * st_error carries only a BORROWED span pointer (src/error.h), so the + * lexer cannot hand back a stack-local span. Instead the lexer stores the + * offending span in its own `err_span` field and passes &lx->err_span to + * st_error_at(). Consequence: the returned error's span is valid only + * until the NEXT st_lexer_next() call or st_lexer_free() — copy the span + * (or print the error) before re-lexing. This mirrors the span.h "file is + * borrowed" convention. + * + * TOKEN-ROLE NOTES (contract for todo 7, the parser) + * --- + * The lexer is deliberately role-agnostic. NODE_NAME / ARGUMENT / PROPERTY + * are parser-level roles, not lexical kinds: the parser classifies + * ST_TOK_IDENT as a node name, an argument value, or a property key + * followed by ST_TOK_EQUALS, by position. String/raw/multi-line text is + * the raw source slice (delimiters and escapes intact) — unescaping and + * multi-line dedenting belong to the value model (todo 8), not here. + * + * Copyright (c) 2026 huntedbytheirs + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ST_KDL_LEXER_H +#define ST_KDL_LEXER_H + +#include + +#include "span.h" + +struct st_error; + +/* Token kinds, ordered so the punctuation kinds follow ST_TOK_ERROR. */ +enum st_token_kind { + ST_TOK_EOF = 0, /* end of input */ + ST_TOK_ERROR, /* lex error; only ever RETURNED, never stored in a token */ + + /* structural punctuation */ + ST_TOK_LBRACE, /* { children-block open */ + ST_TOK_RBRACE, /* } children-block close */ + ST_TOK_SEMICOLON, /* ; node separator */ + ST_TOK_EQUALS, /* = property assignment */ + + /* strings (KDL §3.9) */ + ST_TOK_IDENT, /* bare identifier (node name / arg / property key) */ + ST_TOK_STRING, /* "quoted" single-line string */ + ST_TOK_MULTILINE_STRING, /* """multi-line""" string */ + ST_TOK_RAW_STRING, /* #"raw"# or #"""raw multi-line"""# */ + + /* numbers (KDL §3.14) */ + ST_TOK_NUMBER, /* decimal / 0x hex / 0o octal / 0b binary */ + ST_TOK_INF, /* #inf */ + ST_TOK_NEG_INF, /* #-inf */ + ST_TOK_NAN, /* #nan */ + + /* booleans and null (KDL §3.15-3.16) */ + ST_TOK_TRUE, /* #true */ + ST_TOK_FALSE, /* #false */ + ST_TOK_NULL, /* #null */ + + /* type annotation (KDL §3.8); text slice is the inner identifier */ + ST_TOK_TYPE, /* (u8) -> text "u8" */ + + /* comments (KDL §3.17) */ + ST_TOK_LINE_COMMENT, /* // ... */ + ST_TOK_BLOCK_COMMENT, /* ... (possibly nested) */ + ST_TOK_SLASHDASH, /* /- node/entry-level comment */ +}; + +/* One token. `span` is the 1-based line/col of the token's first byte; + * `text`/`len` are a borrowed slice into the source (NOT NUL-terminated; + * for strings it includes the delimiters). */ +struct st_token { + enum st_token_kind kind; + struct st_span span; + const char *text; + size_t len; +}; + +struct st_lexer; + +struct st_lexer *st_lexer_new(const char *src, const char *filename); +void st_lexer_free(struct st_lexer *lx); + +enum st_token_kind st_lexer_next(struct st_lexer *lx, struct st_token *out, + struct st_error **err); + +/* Stable display name for a token kind (e.g. "ident"). Unknown kinds + * yield "?" rather than indexing out of range. */ +const char *st_token_kind_name(enum st_token_kind kind); + +#endif /* ST_KDL_LEXER_H */ diff --git a/tests/unit/test_lexer.c b/tests/unit/test_lexer.c new file mode 100644 index 0000000..d78a5e6 --- /dev/null +++ b/tests/unit/test_lexer.c @@ -0,0 +1,624 @@ +/* LINK: ../../src/kdl/lexer.c ../../src/error.c ../../src/span.c */ +/* tests/unit/test_lexer.c + * + * Unit tests for the KDL 2.0.0 tokenizer (todo 6). + * + * The magic LINK comment on line 1 is REQUIRED by tests/run.sh: it lists + * the extra .c sources to compile into this test binary (paths relative + * to tests/unit/, space-separated). lexer.c depends on error.c (typed + * errors) and span.c (caret rendering), so all three are linked. + * + * These tests assert on REAL token streams — kind, span (line/col) and + * the text slice — never just "does not crash". + */ +#include "munit.h" + +#include "error.h" +#include "kdl/lexer.h" + +#include +#include +#include + +#define MAX_TOKS 256 +#define MAX_TEXT 512 + +/* A single captured token. `text` is a NUL-terminated copy of the lexer's + * text slice (which is otherwise not NUL-terminated). */ +struct captured_tok { + enum st_token_kind kind; + size_t line; + size_t col; + char text[MAX_TEXT]; +}; + +/* A full captured stream. When the lexer hits an error, `had_error` is + * set and the error's category + span + message are copied out (values, + * not borrowed pointers) before the lexer is freed — this sidesteps the + * borrowed-span lifetime entirely inside the test. */ +struct capture { + struct captured_tok toks[MAX_TOKS]; + size_t n; + bool had_error; + int err_category; + size_t err_line; + size_t err_col; + char err_msg[MAX_TEXT]; +}; + +/* Tokenize `src` to completion. On a lex error the tokens produced before + * the error are still captured (and the error info is recorded). */ +static struct capture +capture(const char *src) +{ + struct capture cap; + struct st_lexer *lx; + struct st_error *err = NULL; + + memset(&cap, 0, sizeof cap); + lx = st_lexer_new(src, "test.kdl"); + munit_assert_not_null(lx); + + for (;;) { + struct st_token t; + enum st_token_kind k = st_lexer_next(lx, &t, &err); + + if (k == ST_TOK_ERROR) { + cap.had_error = true; + if (err != NULL) { + cap.err_category = (int)st_error_category_of(err); + if (err->span != NULL) { + cap.err_line = err->span->line; + cap.err_col = err->span->col; + } + snprintf(cap.err_msg, sizeof cap.err_msg, "%s", + st_error_message(err)); + st_error_free(err); + } + break; + } + + munit_assert_size(cap.n, <, MAX_TOKS); + cap.toks[cap.n].kind = k; + cap.toks[cap.n].line = t.span.line; + cap.toks[cap.n].col = t.span.col; + { + size_t len = t.len < MAX_TEXT - 1 ? t.len : MAX_TEXT - 1; + memcpy(cap.toks[cap.n].text, t.text, len); + cap.toks[cap.n].text[len] = '\0'; + } + cap.n++; + + if (k == ST_TOK_EOF) { + break; + } + } + + st_lexer_free(lx); + return cap; +} + +/* Assert token i is (kind, line, col, text). */ +static void +assert_tok(const struct capture *cap, size_t i, enum st_token_kind kind, + size_t line, size_t col, const char *text) +{ + munit_assert_size(i, <, cap->n); + munit_assert_int(cap->toks[i].kind, ==, kind); + munit_assert_size(cap->toks[i].line, ==, line); + munit_assert_size(cap->toks[i].col, ==, col); + munit_assert_string_equal(cap->toks[i].text, text); +} + +/* Assert a lex error occurred with the given (line, col) span. */ +static void +assert_error(const struct capture *cap, size_t line, size_t col) +{ + munit_assert_true(cap->had_error); + munit_assert_int(cap->err_category, ==, ST_ERR_KDL_PARSE); + munit_assert_size(cap->err_line, ==, line); + munit_assert_size(cap->err_col, ==, col); +} + +/* --- acceptance: the plan's canonical node ----------------------------- */ +static MunitResult +test_acceptance_node(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("node 1 2 \"x\" key=\"v\" { child }"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node"); + assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "1"); + assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 8, "2"); + assert_tok(&cap, 3, ST_TOK_STRING, 1, 10, "\"x\""); + assert_tok(&cap, 4, ST_TOK_IDENT, 1, 14, "key"); + assert_tok(&cap, 5, ST_TOK_EQUALS, 1, 17, "="); + assert_tok(&cap, 6, ST_TOK_STRING, 1, 18, "\"v\""); + assert_tok(&cap, 7, ST_TOK_LBRACE, 1, 22, "{"); + assert_tok(&cap, 8, ST_TOK_IDENT, 1, 24, "child"); + assert_tok(&cap, 9, ST_TOK_RBRACE, 1, 30, "}"); + assert_tok(&cap, 10, ST_TOK_EOF, 1, 31, ""); + munit_assert_size(cap.n, ==, 11); + return MUNIT_OK; +} + +/* --- numbers: every radix + sign/float/exponent/underscore ------------ */ +static MunitResult +test_numbers(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("0x10 0o17 0b101 123 3.14 1e10 1_000"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "0x10"); + assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "0o17"); + assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 11, "0b101"); + assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 17, "123"); + assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 21, "3.14"); + assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 26, "1e10"); + assert_tok(&cap, 6, ST_TOK_NUMBER, 1, 31, "1_000"); + assert_tok(&cap, 7, ST_TOK_EOF, 1, 36, ""); + return MUNIT_OK; +} + +static MunitResult +test_signed_and_radix_numbers(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("-5 +7 -0x10 0XFF 0xff 1e-3"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "-5"); + assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 4, "+7"); + assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 7, "-0x10"); + assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 13, "0XFF"); + assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 18, "0xff"); + assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 23, "1e-3"); + assert_tok(&cap, 6, ST_TOK_EOF, 1, 27, ""); + return MUNIT_OK; +} + +/* --- keyword numbers, booleans, null ---------------------------------- */ +static MunitResult +test_keywords(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("#inf #-inf #nan #true #false #null"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_INF, 1, 1, "#inf"); + assert_tok(&cap, 1, ST_TOK_NEG_INF, 1, 6, "#-inf"); + assert_tok(&cap, 2, ST_TOK_NAN, 1, 12, "#nan"); + assert_tok(&cap, 3, ST_TOK_TRUE, 1, 17, "#true"); + assert_tok(&cap, 4, ST_TOK_FALSE, 1, 23, "#false"); + assert_tok(&cap, 5, ST_TOK_NULL, 1, 30, "#null"); + assert_tok(&cap, 6, ST_TOK_EOF, 1, 35, ""); + return MUNIT_OK; +} + +/* --- type annotations -------------------------------------------------- */ +static MunitResult +test_type_annotation(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("(u8)42 (published)date"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_TYPE, 1, 1, "u8"); + assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 5, "42"); + assert_tok(&cap, 2, ST_TOK_TYPE, 1, 8, "published"); + assert_tok(&cap, 3, ST_TOK_IDENT, 1, 19, "date"); + assert_tok(&cap, 4, ST_TOK_EOF, 1, 23, ""); + return MUNIT_OK; +} + +/* --- string forms: quoted, raw (single + double hash) ------------------ */ +static MunitResult +test_strings_quoted_raw(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("\"hello\" #\"raw\"# ##\"raw2\"##"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_STRING, 1, 1, "\"hello\""); + assert_tok(&cap, 1, ST_TOK_RAW_STRING, 1, 9, "#\"raw\"#"); + assert_tok(&cap, 2, ST_TOK_RAW_STRING, 1, 17, "##\"raw2\"##"); + assert_tok(&cap, 3, ST_TOK_EOF, 1, 27, ""); + return MUNIT_OK; +} + +/* --- multi-line string (delimiters only; dedent rules are todo 8) ------ */ +static MunitResult +test_multiline_string(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("\"\"\"multi\"\"\""); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"multi\"\"\""); + assert_tok(&cap, 1, ST_TOK_EOF, 1, 12, ""); + return MUNIT_OK; +} + +/* --- comments ---------------------------------------------------------- */ +static MunitResult +test_line_comment(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("a // comment\nb"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a"); + assert_tok(&cap, 1, ST_TOK_LINE_COMMENT, 1, 3, "// comment"); + assert_tok(&cap, 2, ST_TOK_IDENT, 2, 1, "b"); + assert_tok(&cap, 3, ST_TOK_EOF, 2, 2, ""); + return MUNIT_OK; +} + +static MunitResult +test_block_comment_nested(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("a /* x /* n */ */ b"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a"); + assert_tok(&cap, 1, ST_TOK_BLOCK_COMMENT, 1, 3, "/* x /* n */ */"); + assert_tok(&cap, 2, ST_TOK_IDENT, 1, 19, "b"); + assert_tok(&cap, 3, ST_TOK_EOF, 1, 20, ""); + return MUNIT_OK; +} + +static MunitResult +test_slashdash(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("/- node"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_SLASHDASH, 1, 1, "/-"); + assert_tok(&cap, 1, ST_TOK_IDENT, 1, 4, "node"); + assert_tok(&cap, 2, ST_TOK_EOF, 1, 8, ""); + return MUNIT_OK; +} + +/* --- structural tokens -------------------------------------------------- */ +static MunitResult +test_semicolons_braces(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("a;b {c}"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a"); + assert_tok(&cap, 1, ST_TOK_SEMICOLON, 1, 2, ";"); + assert_tok(&cap, 2, ST_TOK_IDENT, 1, 3, "b"); + assert_tok(&cap, 3, ST_TOK_LBRACE, 1, 5, "{"); + assert_tok(&cap, 4, ST_TOK_IDENT, 1, 6, "c"); + assert_tok(&cap, 5, ST_TOK_RBRACE, 1, 7, "}"); + assert_tok(&cap, 6, ST_TOK_EOF, 1, 8, ""); + return MUNIT_OK; +} + +/* --- identifiers: sign / dotted / hyphen forms ------------------------- */ +static MunitResult +test_ident_specials(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("--this .md foo-bar a+b _under"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "--this"); + assert_tok(&cap, 1, ST_TOK_IDENT, 1, 8, ".md"); + assert_tok(&cap, 2, ST_TOK_IDENT, 1, 12, "foo-bar"); + assert_tok(&cap, 3, ST_TOK_IDENT, 1, 20, "a+b"); + assert_tok(&cap, 4, ST_TOK_IDENT, 1, 24, "_under"); + assert_tok(&cap, 5, ST_TOK_EOF, 1, 30, ""); + return MUNIT_OK; +} + +/* --- bare keyword identifiers are syntax errors ------------------------ */ +static MunitResult +test_bare_keywords_error(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + static const char *const bare[] = { + "inf", "-inf", "nan", "true", "false", "null", + }; + size_t i; + + for (i = 0; i < sizeof(bare) / sizeof(bare[0]); i++) { + struct capture cap = capture(bare[i]); + assert_error(&cap, 1, 1); + } + + /* also rejected in position, with the span on the offending ident */ + { + struct capture cap = capture("x inf"); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "x"); + assert_error(&cap, 1, 3); + } + return MUNIT_OK; +} + +/* --- unterminated strings (all three forms) ---------------------------- */ +static MunitResult +test_unterminated_quoted(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("node \"unterminated"); + + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node"); + assert_error(&cap, 1, 6); /* span at the opening quote */ + return MUNIT_OK; +} + +static MunitResult +test_unterminated_raw(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("#\"unterminated"); + + assert_error(&cap, 1, 1); /* span at the opening '#' */ + return MUNIT_OK; +} + +static MunitResult +test_unterminated_multiline(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("\"\"\"multi"); + + assert_error(&cap, 1, 1); /* span at the opening '\"\"\"' */ + return MUNIT_OK; +} + +/* --- malformed numbers -------------------------------------------------- */ +static MunitResult +test_bad_hex(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("0xZZ"); + + assert_error(&cap, 1, 1); + return MUNIT_OK; +} + +static MunitResult +test_hex_no_digits(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("0x"); + + assert_error(&cap, 1, 1); + return MUNIT_OK; +} + +static MunitResult +test_bad_octal_binary(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("0o8"); + + assert_error(&cap, 1, 1); + + cap = capture("0b2"); + assert_error(&cap, 1, 1); + return MUNIT_OK; +} + +/* --- leading-dot numbers are errors ------------------------------------ */ +static MunitResult +test_dot_number_error(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture(".5"); + + assert_error(&cap, 1, 1); + return MUNIT_OK; +} + +/* --- invalid escape ----------------------------------------------------- */ +static MunitResult +test_invalid_escape(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("\"\\q\""); + + assert_error(&cap, 1, 1); + return MUNIT_OK; +} + +/* --- stray '#' ---------------------------------------------------------- */ +static MunitResult +test_hash_alone(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("#"); + + assert_error(&cap, 1, 1); + + cap = capture("##foo"); + assert_error(&cap, 1, 1); + return MUNIT_OK; +} + +/* --- bad type annotation ------------------------------------------------ */ +static MunitResult +test_bad_type_annotation(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("(u8"); + + assert_error(&cap, 1, 1); + return MUNIT_OK; +} + +/* --- a stray '}' is a token, not a lexer error (nesting is todo 7) ----- */ +static MunitResult +test_stray_brace_is_token(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("}"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_RBRACE, 1, 1, "}"); + assert_tok(&cap, 1, ST_TOK_EOF, 1, 2, ""); + return MUNIT_OK; +} + +/* --- hostile span: column well past 60 stays exact --------------------- */ +static MunitResult +test_long_line_span(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + char src[256]; + + memset(src, ' ', 70); + memcpy(src + 70, "foo", 4); + + { + struct capture cap = capture(src); + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 71, "foo"); + assert_tok(&cap, 1, ST_TOK_EOF, 1, 74, ""); + } + return MUNIT_OK; +} + +/* --- line/col tracking across newlines --------------------------------- */ +static MunitResult +test_line_col_tracking(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + struct capture cap = capture("foo\n bar"); + + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "foo"); + assert_tok(&cap, 1, ST_TOK_IDENT, 2, 3, "bar"); + assert_tok(&cap, 2, ST_TOK_EOF, 2, 6, ""); + + /* a multi-line string advances line/col for following tokens */ + cap = capture("\"\"\"\nx\n\"\"\" tail"); + munit_assert_false(cap.had_error); + assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"\nx\n\"\"\""); + assert_tok(&cap, 1, ST_TOK_IDENT, 3, 5, "tail"); + assert_tok(&cap, 2, ST_TOK_EOF, 3, 9, ""); + return MUNIT_OK; +} + +/* --- token kind names exist and are distinct --------------------------- */ +static MunitResult +test_kind_names(const MunitParameter params[], void *data) +{ + (void)params; + (void)data; + const char *prev = NULL; + int k; + + for (k = ST_TOK_EOF; k <= ST_TOK_SLASHDASH; k++) { + const char *n = st_token_kind_name((enum st_token_kind)k); + munit_assert_not_null(n); + munit_assert_int((int)strlen(n), >, 0); + if (prev != NULL) { + munit_assert_string_not_equal(n, prev); + } + prev = n; + } + return MUNIT_OK; +} + +static MunitTest tests[] = { + { "/lexer/acceptance-node", test_acceptance_node, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/numbers", test_numbers, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/signed-radix-numbers", test_signed_and_radix_numbers, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/keywords", test_keywords, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/type-annotation", test_type_annotation, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/strings-quoted-raw", test_strings_quoted_raw, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/multiline-string", test_multiline_string, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/line-comment", test_line_comment, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/block-comment-nested", test_block_comment_nested, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/slashdash", test_slashdash, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/semicolons-braces", test_semicolons_braces, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/ident-specials", test_ident_specials, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/bare-keywords-error", test_bare_keywords_error, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/unterminated-quoted", test_unterminated_quoted, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/unterminated-raw", test_unterminated_raw, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/unterminated-multiline", test_unterminated_multiline, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/bad-hex", test_bad_hex, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/hex-no-digits", test_hex_no_digits, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/bad-octal-binary", test_bad_octal_binary, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/dot-number-error", test_dot_number_error, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/invalid-escape", test_invalid_escape, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/hash-alone", test_hash_alone, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/bad-type-annotation", test_bad_type_annotation, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/stray-brace-is-token", test_stray_brace_is_token, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/long-line-span", test_long_line_span, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/line-col-tracking", test_line_col_tracking, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { "/lexer/kind-names", test_kind_names, NULL, NULL, + MUNIT_TEST_OPTION_NONE, NULL }, + { NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }, +}; + +static const MunitSuite suite = { + "/lexer", tests, NULL, 1, MUNIT_SUITE_OPTION_NONE, +}; + +int +main(int argc, char *argv[MUNIT_ARRAY_PARAM(argc + 1)]) +{ + return munit_suite_main(&suite, NULL, argc, argv); +}