huntedbytheirs/stupidtools
Template
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(kdl): add KDL lexer with source spans

Browse Source
This commit is contained in:
huntedbytheirs
2026-08-28 20:28:57 -04:00
parent 468aa1c4ef
commit 50afaa17ca
3 changed files with 1686 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/kdl/lexer.c
+943
View File
@@ -0,0 +1,943 @@
/*
* lexer.c - KDL 2.0.0 tokenizer for stupidtools (todo 6).
*
* Produces a FLAT token stream with a source span on every token. It does
* NOT parse node/argument/property structure (the parser does, todo 7).
* See lexer.h for the full API + error-span-lifetime contract.
*
* Design decisions (mirrored in .omo/notepads/stupidtools/learnings.md):
* - identifiers are scanned greedily; non-ASCII bytes are identifier
* characters (the non-identifier set is the ASCII one from §3.10.2).
* - bare `inf` / `-inf` / `nan` / `true` / `false` / `null` are syntax
* errors; the `#`-prefixed keyword forms are distinct token kinds.
* - multi-line strings are delimited by `"""` only; the spec's "first
* line must be a newline" + dedent-prefix rules are VALUE rules and
* are deferred to the value model (todo 8), so `"""multi"""` tokenizes.
* - columns are byte-based (not codepoints); full Unicode grapheme
* columns and disallowed-literal-code-point validation are deferred.
*
* Copyright (c) 2026 huntedbytheirs
* SPDX-License-Identifier: BSD-3-Clause
*/
#include "kdl/lexer.h"
#include "error.h"
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
struct st_lexer {
const char *src; /* borrowed source buffer (NUL-terminated) */
const char *pos; /* current scan position */
const char *filename; /* borrowed; stamped into every span */
size_t line; /* current 1-based line */
size_t col; /* current 1-based column (bytes) */
struct st_span err_span; /* stable span for the most recent error */
};
/*
* Character classes. Non-ASCII bytes are deliberately treated as
* identifier characters (see header note); the classification below only
* special-cases ASCII + the UTF-8 newline sequences (NEL/LS/PS).
*/
static bool
is_digit(unsigned char c)
{
return c >= '0' && c <= '9';
}
static bool
is_hex_digit(unsigned char c)
{
return is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
static bool
is_oct_digit(unsigned char c)
{
return c >= '0' && c <= '7';
}
static bool
is_bin_digit(unsigned char c)
{
return c == '0' || c == '1';
}
/* Non-newline whitespace (ASCII subset; full Unicode White_Space deferred). */
static bool
is_space(unsigned char c)
{
return c == ' ' || c == '\t';
}
/* Non-identifier ASCII chars per KDL §3.10.2: \ / ( ) { } ; [ ] " # = */
static bool
is_nonident(unsigned char c)
{
switch (c) {
case '\\':
case '/':
case '(':
case ')':
case '{':
case '}':
case ';':
case '[':
case ']':
case '"':
case '#':
case '=':
return true;
default:
return false;
}
}
/* Newline sequences per KDL §3.18: CRLF (one newline), CR, LF, VT, FF,
* NEL (U+0085), LS (U+2028), PS (U+2029). */
static bool
is_newline_at(const char *p)
{
unsigned char c = (unsigned char)p[0];
if (c == '\r' || c == '\n' || c == '\v' || c == '\f') {
return true;
}
if (c == 0xC2 && (unsigned char)p[1] == 0x85) {
return true; /* NEL */
}
if (c == 0xE2 && (unsigned char)p[1] == 0x80 &&
((unsigned char)p[2] == 0xA8 || (unsigned char)p[2] == 0xA9)) {
return true; /* LS, PS */
}
return false;
}
/* Advance one character, updating line/col. CRLF counts as one newline. */
static void
advance(struct st_lexer *lx)
{
unsigned char c = (unsigned char)*lx->pos;
if (c == '\0') {
return;
}
if (is_newline_at(lx->pos)) {
if (c == '\r' && lx->pos[1] == '\n') {
lx->pos += 2;
} else if (c == 0xC2) {
lx->pos += 2;
} else if (c == 0xE2) {
lx->pos += 3;
} else {
lx->pos += 1;
}
lx->line++;
lx->col = 1;
return;
}
lx->pos += 1;
lx->col += 1;
}
/* True when the cursor is on a byte that may continue an identifier. */
static bool
at_ident_char(const struct st_lexer *lx)
{
unsigned char c = (unsigned char)*lx->pos;
if (c == '\0' || is_nonident(c) || is_space(c)) {
return false;
}
return !is_newline_at(lx->pos);
}
/* True when the cursor is on a byte that may START an identifier (i.e. not
* a digit, non-ident char, whitespace, or newline). `+`/`-`/`.` are handled
* by the caller because they can also begin numbers. */
static bool
at_ident_start(const struct st_lexer *lx)
{
unsigned char c = (unsigned char)*lx->pos;
if (c == '\0' || is_digit(c) || is_nonident(c) || is_space(c)) {
return false;
}
return !is_newline_at(lx->pos);
}
/* Build a KDL-parse error at `sp` and return ST_TOK_ERROR. The span is
* stored in the lexer so the error's borrowed pointer stays valid until
* the next st_lexer_next/free (see lexer.h). */
static enum st_token_kind
fail(struct st_lexer *lx, struct st_span sp, const char *msg,
struct st_error **err)
{
if (err != NULL) {
lx->err_span = sp;
*err = st_error_at(st_error_kdl_parse(msg), &lx->err_span);
}
return ST_TOK_ERROR;
}
static void
emit(struct st_token *out, enum st_token_kind kind, struct st_span span,
const char *text, size_t len)
{
out->kind = kind;
out->span = span;
out->text = text;
out->len = len;
}
/* Is `s` (len bytes) one of the bare keyword identifiers that KDL forbids
* as an identifier string (must be written with `#` or quoted)? */
static bool
is_bare_keyword(const char *s, size_t len)
{
static const char *const keywords[] = {
"true", "false", "null", "inf", "-inf", "nan",
};
size_t i;
for (i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) {
size_t klen = strlen(keywords[i]);
if (len == klen && memcmp(s, keywords[i], klen) == 0) {
return true;
}
}
return false;
}
/* --- line continuations ------------------------------------------------ */
/* Consume an escline: '\' ws* (single-line-comment | newline | eof).
* Returns 0 on success, -1 on error (*err set). */
static int
lex_escline(struct st_lexer *lx, struct st_error **err)
{
struct st_span sp = { lx->filename, lx->line, lx->col };
advance(lx); /* '\' */
while (*lx->pos != '\0' && is_space((unsigned char)*lx->pos)) {
advance(lx);
}
if (*lx->pos == '\0') {
return 0; /* eof is a legal escline terminator */
}
if (is_newline_at(lx->pos)) {
advance(lx);
return 0;
}
if (lx->pos[0] == '/' && lx->pos[1] == '/') {
advance(lx);
advance(lx);
while (*lx->pos != '\0' && !is_newline_at(lx->pos)) {
advance(lx);
}
if (*lx->pos != '\0') {
advance(lx); /* the terminating newline */
}
return 0;
}
(void)fail(lx, sp,
"invalid line continuation: '\\' must be followed by a newline "
"or a line comment", err);
return -1;
}
/* --- identifiers ------------------------------------------------------- */
static enum st_token_kind
lex_ident(struct st_lexer *lx, struct st_token *out, struct st_span start,
struct st_error **err)
{
const char *begin = lx->pos;
advance(lx); /* first char is a valid ident-start (caller guarantees) */
while (at_ident_char(lx)) {
advance(lx);
}
if (is_bare_keyword(begin, (size_t)(lx->pos - begin))) {
return fail(lx, start,
"bare keyword identifier; write it as a '#'-prefixed keyword "
"or a quoted string", err);
}
emit(out, ST_TOK_IDENT, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_IDENT;
}
/* --- numbers ----------------------------------------------------------- */
static enum st_token_kind
lex_number(struct st_lexer *lx, struct st_token *out, struct st_span start,
struct st_error **err)
{
const char *begin = lx->pos;
if (*lx->pos == '+' || *lx->pos == '-') {
advance(lx); /* sign */
}
if (lx->pos[0] == '0' && (lx->pos[1] == 'x' || lx->pos[1] == 'X')) {
advance(lx);
advance(lx); /* 0x */
if (!is_hex_digit((unsigned char)*lx->pos)) {
return fail(lx, start,
"invalid hexadecimal number: expected a hex digit after "
"'0x'", err);
}
while (is_hex_digit((unsigned char)*lx->pos) ||
*lx->pos == '_') {
advance(lx);
}
} else if (lx->pos[0] == '0' &&
(lx->pos[1] == 'o' || lx->pos[1] == 'O')) {
advance(lx);
advance(lx); /* 0o */
if (!is_oct_digit((unsigned char)*lx->pos)) {
return fail(lx, start,
"invalid octal number: expected an octal digit after '0o'",
err);
}
while (is_oct_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
advance(lx);
}
} else if (lx->pos[0] == '0' &&
(lx->pos[1] == 'b' || lx->pos[1] == 'B')) {
advance(lx);
advance(lx); /* 0b */
if (!is_bin_digit((unsigned char)*lx->pos)) {
return fail(lx, start,
"invalid binary number: expected a binary digit after '0b'",
err);
}
while (is_bin_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
advance(lx);
}
} else {
/* decimal: integer ('.' integer)? exponent? */
while (is_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
advance(lx);
}
if (*lx->pos == '.' && is_digit((unsigned char)lx->pos[1])) {
advance(lx); /* '.' */
while (is_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
advance(lx);
}
}
if (*lx->pos == 'e' || *lx->pos == 'E') {
const char *save = lx->pos;
size_t save_line = lx->line;
size_t save_col = lx->col;
advance(lx); /* e */
if (*lx->pos == '+' || *lx->pos == '-') {
advance(lx);
}
if (!is_digit((unsigned char)*lx->pos)) {
/* not a real exponent: back off */
lx->pos = save;
lx->line = save_line;
lx->col = save_col;
} else {
while (is_digit((unsigned char)*lx->pos) ||
*lx->pos == '_') {
advance(lx);
}
}
}
}
emit(out, ST_TOK_NUMBER, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_NUMBER;
}
/* Dispatch for bytes that may start either a number or an identifier
* (digits, '+', '-', '.'). */
static enum st_token_kind
lex_value_start(struct st_lexer *lx, struct st_token *out,
struct st_span start, struct st_error **err)
{
unsigned char c = (unsigned char)*lx->pos;
if (is_digit(c)) {
return lex_number(lx, out, start, err);
}
if (c == '+' || c == '-') {
unsigned char n = (unsigned char)lx->pos[1];
if (is_digit(n)) {
return lex_number(lx, out, start, err);
}
if (n == '.' && is_digit((unsigned char)lx->pos[2])) {
return fail(lx, start,
"number must have a leading digit before the decimal point",
err);
}
return lex_ident(lx, out, start, err);
}
if (c == '.') {
if (is_digit((unsigned char)lx->pos[1])) {
return fail(lx, start,
"number must have a leading digit (write '0.1', not '.1')",
err);
}
return lex_ident(lx, out, start, err);
}
return lex_ident(lx, out, start, err);
}
/* --- strings ----------------------------------------------------------- */
/* Scan one escape sequence (at a '\'); validates it but does not unescape.
* Returns 0 on success, -1 on error. */
static int
lex_escape(struct st_lexer *lx, struct st_span start, struct st_error **err)
{
unsigned char c;
advance(lx); /* '\' */
c = (unsigned char)*lx->pos;
if (c == '\0') {
(void)fail(lx, start, "unterminated escape sequence in string", err);
return -1;
}
switch (c) {
case 'n':
case 'r':
case 't':
case '\\':
case '"':
case 'b':
case 'f':
case 's':
advance(lx);
return 0;
case 'u': {
int ndigits = 0;
advance(lx); /* u */
if (*lx->pos != '{') {
(void)fail(lx, start,
"invalid unicode escape: expected '{' after '\\u'", err);
return -1;
}
advance(lx); /* { */
while (is_hex_digit((unsigned char)*lx->pos)) {
advance(lx);
ndigits++;
}
if (*lx->pos != '}') {
(void)fail(lx, start,
"invalid unicode escape: expected '}'", err);
return -1;
}
if (ndigits == 0 || ndigits > 6) {
(void)fail(lx, start,
"invalid unicode escape: expected 1-6 hex digits", err);
return -1;
}
advance(lx); /* } */
return 0;
}
default:
/* whitespace escape: '\' + one-or-more whitespace/newlines */
if (is_space(c) || is_newline_at(lx->pos)) {
while (*lx->pos != '\0' &&
(is_space((unsigned char)*lx->pos) ||
is_newline_at(lx->pos))) {
advance(lx);
}
return 0;
}
(void)fail(lx, start, "invalid escape sequence in string", err);
return -1;
}
}
/* Single-line quoted string: "..." (raw disables escapes). */
static enum st_token_kind
lex_quoted_string(struct st_lexer *lx, struct st_token *out,
struct st_span start, struct st_error **err, bool raw)
{
const char *begin = lx->pos;
advance(lx); /* opening '"' */
for (;;) {
unsigned char c = (unsigned char)*lx->pos;
if (c == '\0') {
return fail(lx, start, "unterminated string literal", err);
}
if (c == '"') {
advance(lx); /* closing '"' */
break;
}
if (!raw && c == '\\') {
if (lex_escape(lx, start, err) < 0) {
return ST_TOK_ERROR;
}
continue;
}
if (is_newline_at(lx->pos)) {
return fail(lx, start,
"unterminated string literal (newline before closing quote)",
err);
}
advance(lx);
}
emit(out, ST_TOK_STRING, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_STRING;
}
/* Multi-line string: """...""" (raw disables escapes). Only the delimiters
* are recognized here; dedent/value rules are deferred to todo 8. */
static enum st_token_kind
lex_multiline_string(struct st_lexer *lx, struct st_token *out,
struct st_span start, struct st_error **err, bool raw)
{
const char *begin = lx->pos;
advance(lx);
advance(lx);
advance(lx); /* opening '"""' */
for (;;) {
if (*lx->pos == '\0') {
return fail(lx, start, "unterminated multi-line string", err);
}
if (lx->pos[0] == '"' && lx->pos[1] == '"' && lx->pos[2] == '"') {
advance(lx);
advance(lx);
advance(lx); /* closing '"""' */
break;
}
if (!raw && *lx->pos == '\\') {
if (lex_escape(lx, start, err) < 0) {
return ST_TOK_ERROR;
}
continue;
}
advance(lx);
}
emit(out, ST_TOK_MULTILINE_STRING, start, begin,
(size_t)(lx->pos - begin));
return ST_TOK_MULTILINE_STRING;
}
/* Raw string: '#'* '"' body '"' '#'* or '#'* '"""' body '"""' '#'*.
* The body may contain '"' / '#' as long as it is not the closing
* delimiter (the closing quote followed by exactly n '#'). */
static enum st_token_kind
lex_raw_string(struct st_lexer *lx, struct st_token *out,
struct st_span start, struct st_error **err)
{
const char *begin = lx->pos;
size_t n = 0;
size_t i;
bool multi;
while (*lx->pos == '#') {
advance(lx);
n++;
}
multi = (lx->pos[0] == '"' && lx->pos[1] == '"' && lx->pos[2] == '"');
if (multi) {
advance(lx);
advance(lx);
advance(lx); /* '"""' */
for (;;) {
if (*lx->pos == '\0') {
return fail(lx, start,
"unterminated raw multi-line string", err);
}
if (lx->pos[0] == '"' && lx->pos[1] == '"' &&
lx->pos[2] == '"') {
bool closes = true;
for (i = 0; i < n; i++) {
if (lx->pos[3 + i] != '#') {
closes = false;
break;
}
}
if (closes) {
advance(lx);
advance(lx);
advance(lx);
for (i = 0; i < n; i++) {
advance(lx);
}
break;
}
}
advance(lx);
}
} else {
advance(lx); /* '"' */
for (;;) {
if (*lx->pos == '\0') {
return fail(lx, start,
"unterminated raw string literal", err);
}
if (lx->pos[0] == '"') {
bool closes = true;
for (i = 0; i < n; i++) {
if (lx->pos[1 + i] != '#') {
closes = false;
break;
}
}
if (closes) {
advance(lx); /* '"' */
for (i = 0; i < n; i++) {
advance(lx);
}
break;
}
}
if (is_newline_at(lx->pos)) {
return fail(lx, start,
"unterminated raw string literal (newline before "
"closing quote)", err);
}
advance(lx);
}
}
emit(out, ST_TOK_RAW_STRING, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_RAW_STRING;
}
/* --- keywords (#inf / #-inf / #nan / #true / #false / #null) ----------- */
static enum st_token_kind
lex_keyword(struct st_lexer *lx, struct st_token *out, struct st_span start,
struct st_error **err)
{
const char *begin = lx->pos;
const char *kw;
size_t kwlen;
advance(lx); /* '#' */
kw = lx->pos;
if (*lx->pos == '-') {
advance(lx);
}
while (at_ident_char(lx)) {
advance(lx);
}
kwlen = (size_t)(lx->pos - kw);
if (kwlen == 3 && memcmp(kw, "inf", 3) == 0) {
emit(out, ST_TOK_INF, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_INF;
}
if (kwlen == 4 && memcmp(kw, "-inf", 4) == 0) {
emit(out, ST_TOK_NEG_INF, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_NEG_INF;
}
if (kwlen == 3 && memcmp(kw, "nan", 3) == 0) {
emit(out, ST_TOK_NAN, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_NAN;
}
if (kwlen == 4 && memcmp(kw, "true", 4) == 0) {
emit(out, ST_TOK_TRUE, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_TRUE;
}
if (kwlen == 5 && memcmp(kw, "false", 5) == 0) {
emit(out, ST_TOK_FALSE, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_FALSE;
}
if (kwlen == 4 && memcmp(kw, "null", 4) == 0) {
emit(out, ST_TOK_NULL, start, begin, (size_t)(lx->pos - begin));
return ST_TOK_NULL;
}
return fail(lx, start, "unknown '#'-prefixed keyword", err);
}
/* Dispatch on '#': raw string or keyword. */
static enum st_token_kind
lex_hash(struct st_lexer *lx, struct st_token *out, struct st_span start,
struct st_error **err)
{
const char *p = lx->pos;
while (*p == '#') {
p++;
}
if (*p == '"') {
return lex_raw_string(lx, out, start, err);
}
if (lx->pos[1] == '#') {
return fail(lx, start,
"invalid token: '#' must be followed by a string or a keyword",
err);
}
return lex_keyword(lx, out, start, err);
}
/* --- type annotation --------------------------------------------------- */
static enum st_token_kind
lex_type(struct st_lexer *lx, struct st_token *out, struct st_span start,
struct st_error **err)
{
const char *name;
size_t namelen;
advance(lx); /* '(' */
while (is_space((unsigned char)*lx->pos)) {
advance(lx);
}
if (!at_ident_start(lx)) {
return fail(lx, start, "expected a type name inside '('", err);
}
name = lx->pos;
advance(lx);
while (at_ident_char(lx)) {
advance(lx);
}
namelen = (size_t)(lx->pos - name);
while (is_space((unsigned char)*lx->pos)) {
advance(lx);
}
if (*lx->pos != ')') {
return fail(lx, start,
"unterminated type annotation (expected ')')", err);
}
advance(lx); /* ')' */
emit(out, ST_TOK_TYPE, start, name, namelen);
return ST_TOK_TYPE;
}
/* --- comments ---------------------------------------------------------- */
static enum st_token_kind
lex_comment(struct st_lexer *lx, struct st_token *out, struct st_span start,
struct st_error **err)
{
const char *begin = lx->pos;
if (lx->pos[1] == '/') {
advance(lx);
advance(lx); /* '//' */
while (*lx->pos != '\0' && !is_newline_at(lx->pos)) {
advance(lx);
}
emit(out, ST_TOK_LINE_COMMENT, start, begin,
(size_t)(lx->pos - begin));
return ST_TOK_LINE_COMMENT;
}
if (lx->pos[1] == '*') {
int depth = 1;
advance(lx);
advance(lx); /* slash-star opener */
while (depth > 0) {
if (*lx->pos == '\0') {
return fail(lx, start, "unterminated block comment", err);
}
if (lx->pos[0] == '/' && lx->pos[1] == '*') {
advance(lx);
advance(lx);
depth++;
continue;
}
if (lx->pos[0] == '*' && lx->pos[1] == '/') {
advance(lx);
advance(lx);
depth--;
continue;
}
advance(lx);
}
emit(out, ST_TOK_BLOCK_COMMENT, start, begin,
(size_t)(lx->pos - begin));
return ST_TOK_BLOCK_COMMENT;
}
if (lx->pos[1] == '-') {
advance(lx);
advance(lx); /* '/-' */
emit(out, ST_TOK_SLASHDASH, start, begin,
(size_t)(lx->pos - begin));
return ST_TOK_SLASHDASH;
}
return fail(lx, start, "unexpected '/'", err);
}
/* --- public API -------------------------------------------------------- */
struct st_lexer *
st_lexer_new(const char *src, const char *filename)
{
struct st_lexer *lx = calloc(1, sizeof(*lx));
if (lx == NULL) {
return NULL;
}
lx->src = src != NULL ? src : "";
lx->pos = lx->src;
lx->filename = filename;
lx->line = 1;
lx->col = 1;
/* Skip a leading BOM (U+FEFF = EF BB BF), legal only as the very first
* code point of a document. */
if ((unsigned char)lx->pos[0] == 0xEF &&
(unsigned char)lx->pos[1] == 0xBB &&
(unsigned char)lx->pos[2] == 0xBF) {
lx->pos += 3;
}
return lx;
}
void
st_lexer_free(struct st_lexer *lx)
{
free(lx);
}
enum st_token_kind
st_lexer_next(struct st_lexer *lx, struct st_token *out, struct st_error **err)
{
unsigned char c;
struct st_span start;
if (err != NULL) {
*err = NULL;
}
/* Skip whitespace, newlines, and line continuations. */
for (;;) {
while (*lx->pos != '\0' &&
(is_space((unsigned char)*lx->pos) || is_newline_at(lx->pos))) {
advance(lx);
}
if (*lx->pos == '\\') {
if (lex_escline(lx, err) < 0) {
return ST_TOK_ERROR;
}
continue;
}
break;
}
if (*lx->pos == '\0') {
emit(out, ST_TOK_EOF, (struct st_span){ lx->filename, lx->line,
lx->col }, lx->pos, 0);
return ST_TOK_EOF;
}
start = (struct st_span){ lx->filename, lx->line, lx->col };
c = (unsigned char)*lx->pos;
/* numbers, signs, and dotted identifiers */
if (is_digit(c) || c == '+' || c == '-' || c == '.') {
return lex_value_start(lx, out, start, err);
}
switch (c) {
case '{':
emit(out, ST_TOK_LBRACE, start, lx->pos, 1);
advance(lx);
break;
case '}':
emit(out, ST_TOK_RBRACE, start, lx->pos, 1);
advance(lx);
break;
case ';':
emit(out, ST_TOK_SEMICOLON, start, lx->pos, 1);
advance(lx);
break;
case '=':
emit(out, ST_TOK_EQUALS, start, lx->pos, 1);
advance(lx);
break;
case '"':
if (lx->pos[1] == '"' && lx->pos[2] == '"') {
return lex_multiline_string(lx, out, start, err, false);
}
return lex_quoted_string(lx, out, start, err, false);
case '#':
return lex_hash(lx, out, start, err);
case '(':
return lex_type(lx, out, start, err);
case '/':
return lex_comment(lx, out, start, err);
default:
if (at_ident_start(lx)) {
return lex_ident(lx, out, start, err);
}
return fail(lx, start, "unexpected character in source", err);
}
return out->kind;
}
const char *
st_token_kind_name(enum st_token_kind kind)
{
switch (kind) {
case ST_TOK_EOF:
return "eof";
case ST_TOK_ERROR:
return "error";
case ST_TOK_LBRACE:
return "lbrace";
case ST_TOK_RBRACE:
return "rbrace";
case ST_TOK_SEMICOLON:
return "semicolon";
case ST_TOK_EQUALS:
return "equals";
case ST_TOK_IDENT:
return "ident";
case ST_TOK_STRING:
return "string";
case ST_TOK_MULTILINE_STRING:
return "multiline-string";
case ST_TOK_RAW_STRING:
return "raw-string";
case ST_TOK_NUMBER:
return "number";
case ST_TOK_INF:
return "inf";
case ST_TOK_NEG_INF:
return "-inf";
case ST_TOK_NAN:
return "nan";
case ST_TOK_TRUE:
return "true";
case ST_TOK_FALSE:
return "false";
case ST_TOK_NULL:
return "null";
case ST_TOK_TYPE:
return "type";
case ST_TOK_LINE_COMMENT:
return "line-comment";
case ST_TOK_BLOCK_COMMENT:
return "block-comment";
case ST_TOK_SLASHDASH:
return "slashdash";
default:
return "?";
}
}
src/kdl/lexer.h
+119
View File
@@ -0,0 +1,119 @@
/*
* lexer.h - KDL 2.0.0 tokenizer for stupidtools.
*
* Produces a FLAT token stream; it does NOT parse node/argument/property
* structure (that is the parser, todo 7). Every token carries a source
* span (struct st_span) and a borrowed text slice into the source buffer.
*
* API
* ---
* struct st_lexer *st_lexer_new(const char *src, const char *filename);
* enum st_token_kind st_lexer_next(struct st_lexer *, struct st_token *,
* struct st_error **err);
* void st_lexer_free(struct st_lexer *);
* const char *st_token_kind_name(enum st_token_kind);
*
* st_lexer_new() borrows `src` (never copies); the buffer must remain
* valid for the lexer's lifetime because token text slices point into it.
* `filename` is also borrowed and is stamped into every span.
*
* st_lexer_next() returns one token per call. It returns:
* - a token kind (>= ST_TOK_LBRACE) and fills `*out` on success,
* - ST_TOK_EOF at end of input (out->text/len are empty),
* - ST_TOK_ERROR on a lex error, in which case `*err` is set to a
* freshly allocated st_error (category ST_ERR_KDL_PARSE) and `*out`
* is left unchanged. The caller owns `*err` and frees it with
* st_error_free().
*
* ERROR-SPAN LIFETIME (borrowed span)
* ---
* st_error carries only a BORROWED span pointer (src/error.h), so the
* lexer cannot hand back a stack-local span. Instead the lexer stores the
* offending span in its own `err_span` field and passes &lx->err_span to
* st_error_at(). Consequence: the returned error's span is valid only
* until the NEXT st_lexer_next() call or st_lexer_free() — copy the span
* (or print the error) before re-lexing. This mirrors the span.h "file is
* borrowed" convention.
*
* TOKEN-ROLE NOTES (contract for todo 7, the parser)
* ---
* The lexer is deliberately role-agnostic. NODE_NAME / ARGUMENT / PROPERTY
* are parser-level roles, not lexical kinds: the parser classifies
* ST_TOK_IDENT as a node name, an argument value, or a property key
* followed by ST_TOK_EQUALS, by position. String/raw/multi-line text is
* the raw source slice (delimiters and escapes intact) — unescaping and
* multi-line dedenting belong to the value model (todo 8), not here.
*
* Copyright (c) 2026 huntedbytheirs
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef ST_KDL_LEXER_H
#define ST_KDL_LEXER_H
#include <stddef.h>
#include "span.h"
struct st_error;
/* Token kinds, ordered so the punctuation kinds follow ST_TOK_ERROR. */
enum st_token_kind {
ST_TOK_EOF = 0, /* end of input */
ST_TOK_ERROR, /* lex error; only ever RETURNED, never stored in a token */
/* structural punctuation */
ST_TOK_LBRACE, /* { children-block open */
ST_TOK_RBRACE, /* } children-block close */
ST_TOK_SEMICOLON, /* ; node separator */
ST_TOK_EQUALS, /* = property assignment */
/* strings (KDL §3.9) */
ST_TOK_IDENT, /* bare identifier (node name / arg / property key) */
ST_TOK_STRING, /* "quoted" single-line string */
ST_TOK_MULTILINE_STRING, /* """multi-line""" string */
ST_TOK_RAW_STRING, /* #"raw"# or #"""raw multi-line"""# */
/* numbers (KDL §3.14) */
ST_TOK_NUMBER, /* decimal / 0x hex / 0o octal / 0b binary */
ST_TOK_INF, /* #inf */
ST_TOK_NEG_INF, /* #-inf */
ST_TOK_NAN, /* #nan */
/* booleans and null (KDL §3.15-3.16) */
ST_TOK_TRUE, /* #true */
ST_TOK_FALSE, /* #false */
ST_TOK_NULL, /* #null */
/* type annotation (KDL §3.8); text slice is the inner identifier */
ST_TOK_TYPE, /* (u8) -> text "u8" */
/* comments (KDL §3.17) */
ST_TOK_LINE_COMMENT, /* // ... */
ST_TOK_BLOCK_COMMENT, /* ... (possibly nested) */
ST_TOK_SLASHDASH, /* /- node/entry-level comment */
};
/* One token. `span` is the 1-based line/col of the token's first byte;
* `text`/`len` are a borrowed slice into the source (NOT NUL-terminated;
* for strings it includes the delimiters). */
struct st_token {
enum st_token_kind kind;
struct st_span span;
const char *text;
size_t len;
};
struct st_lexer;
struct st_lexer *st_lexer_new(const char *src, const char *filename);
void st_lexer_free(struct st_lexer *lx);
enum st_token_kind st_lexer_next(struct st_lexer *lx, struct st_token *out,
struct st_error **err);
/* Stable display name for a token kind (e.g. "ident"). Unknown kinds
* yield "?" rather than indexing out of range. */
const char *st_token_kind_name(enum st_token_kind kind);
#endif /* ST_KDL_LEXER_H */
tests/unit/test_lexer.c
+624
View File
@@ -0,0 +1,624 @@
/* LINK: ../../src/kdl/lexer.c ../../src/error.c ../../src/span.c */
/* tests/unit/test_lexer.c
*
* Unit tests for the KDL 2.0.0 tokenizer (todo 6).
*
* The magic LINK comment on line 1 is REQUIRED by tests/run.sh: it lists
* the extra .c sources to compile into this test binary (paths relative
* to tests/unit/, space-separated). lexer.c depends on error.c (typed
* errors) and span.c (caret rendering), so all three are linked.
*
* These tests assert on REAL token streams — kind, span (line/col) and
* the text slice — never just "does not crash".
*/
#include "munit.h"
#include "error.h"
#include "kdl/lexer.h"
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#define MAX_TOKS 256
#define MAX_TEXT 512
/* A single captured token. `text` is a NUL-terminated copy of the lexer's
* text slice (which is otherwise not NUL-terminated). */
struct captured_tok {
enum st_token_kind kind;
size_t line;
size_t col;
char text[MAX_TEXT];
};
/* A full captured stream. When the lexer hits an error, `had_error` is
* set and the error's category + span + message are copied out (values,
* not borrowed pointers) before the lexer is freed — this sidesteps the
* borrowed-span lifetime entirely inside the test. */
struct capture {
struct captured_tok toks[MAX_TOKS];
size_t n;
bool had_error;
int err_category;
size_t err_line;
size_t err_col;
char err_msg[MAX_TEXT];
};
/* Tokenize `src` to completion. On a lex error the tokens produced before
* the error are still captured (and the error info is recorded). */
static struct capture
capture(const char *src)
{
struct capture cap;
struct st_lexer *lx;
struct st_error *err = NULL;
memset(&cap, 0, sizeof cap);
lx = st_lexer_new(src, "test.kdl");
munit_assert_not_null(lx);
for (;;) {
struct st_token t;
enum st_token_kind k = st_lexer_next(lx, &t, &err);
if (k == ST_TOK_ERROR) {
cap.had_error = true;
if (err != NULL) {
cap.err_category = (int)st_error_category_of(err);
if (err->span != NULL) {
cap.err_line = err->span->line;
cap.err_col = err->span->col;
}
snprintf(cap.err_msg, sizeof cap.err_msg, "%s",
st_error_message(err));
st_error_free(err);
}
break;
}
munit_assert_size(cap.n, <, MAX_TOKS);
cap.toks[cap.n].kind = k;
cap.toks[cap.n].line = t.span.line;
cap.toks[cap.n].col = t.span.col;
{
size_t len = t.len < MAX_TEXT - 1 ? t.len : MAX_TEXT - 1;
memcpy(cap.toks[cap.n].text, t.text, len);
cap.toks[cap.n].text[len] = '\0';
}
cap.n++;
if (k == ST_TOK_EOF) {
break;
}
}
st_lexer_free(lx);
return cap;
}
/* Assert token i is (kind, line, col, text). */
static void
assert_tok(const struct capture *cap, size_t i, enum st_token_kind kind,
size_t line, size_t col, const char *text)
{
munit_assert_size(i, <, cap->n);
munit_assert_int(cap->toks[i].kind, ==, kind);
munit_assert_size(cap->toks[i].line, ==, line);
munit_assert_size(cap->toks[i].col, ==, col);
munit_assert_string_equal(cap->toks[i].text, text);
}
/* Assert a lex error occurred with the given (line, col) span. */
static void
assert_error(const struct capture *cap, size_t line, size_t col)
{
munit_assert_true(cap->had_error);
munit_assert_int(cap->err_category, ==, ST_ERR_KDL_PARSE);
munit_assert_size(cap->err_line, ==, line);
munit_assert_size(cap->err_col, ==, col);
}
/* --- acceptance: the plan's canonical node ----------------------------- */
static MunitResult
test_acceptance_node(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("node 1 2 \"x\" key=\"v\" { child }");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "1");
assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 8, "2");
assert_tok(&cap, 3, ST_TOK_STRING, 1, 10, "\"x\"");
assert_tok(&cap, 4, ST_TOK_IDENT, 1, 14, "key");
assert_tok(&cap, 5, ST_TOK_EQUALS, 1, 17, "=");
assert_tok(&cap, 6, ST_TOK_STRING, 1, 18, "\"v\"");
assert_tok(&cap, 7, ST_TOK_LBRACE, 1, 22, "{");
assert_tok(&cap, 8, ST_TOK_IDENT, 1, 24, "child");
assert_tok(&cap, 9, ST_TOK_RBRACE, 1, 30, "}");
assert_tok(&cap, 10, ST_TOK_EOF, 1, 31, "");
munit_assert_size(cap.n, ==, 11);
return MUNIT_OK;
}
/* --- numbers: every radix + sign/float/exponent/underscore ------------ */
static MunitResult
test_numbers(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0x10 0o17 0b101 123 3.14 1e10 1_000");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "0x10");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "0o17");
assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 11, "0b101");
assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 17, "123");
assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 21, "3.14");
assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 26, "1e10");
assert_tok(&cap, 6, ST_TOK_NUMBER, 1, 31, "1_000");
assert_tok(&cap, 7, ST_TOK_EOF, 1, 36, "");
return MUNIT_OK;
}
static MunitResult
test_signed_and_radix_numbers(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("-5 +7 -0x10 0XFF 0xff 1e-3");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "-5");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 4, "+7");
assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 7, "-0x10");
assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 13, "0XFF");
assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 18, "0xff");
assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 23, "1e-3");
assert_tok(&cap, 6, ST_TOK_EOF, 1, 27, "");
return MUNIT_OK;
}
/* --- keyword numbers, booleans, null ---------------------------------- */
static MunitResult
test_keywords(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("#inf #-inf #nan #true #false #null");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_INF, 1, 1, "#inf");
assert_tok(&cap, 1, ST_TOK_NEG_INF, 1, 6, "#-inf");
assert_tok(&cap, 2, ST_TOK_NAN, 1, 12, "#nan");
assert_tok(&cap, 3, ST_TOK_TRUE, 1, 17, "#true");
assert_tok(&cap, 4, ST_TOK_FALSE, 1, 23, "#false");
assert_tok(&cap, 5, ST_TOK_NULL, 1, 30, "#null");
assert_tok(&cap, 6, ST_TOK_EOF, 1, 35, "");
return MUNIT_OK;
}
/* --- type annotations -------------------------------------------------- */
static MunitResult
test_type_annotation(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("(u8)42 (published)date");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_TYPE, 1, 1, "u8");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 5, "42");
assert_tok(&cap, 2, ST_TOK_TYPE, 1, 8, "published");
assert_tok(&cap, 3, ST_TOK_IDENT, 1, 19, "date");
assert_tok(&cap, 4, ST_TOK_EOF, 1, 23, "");
return MUNIT_OK;
}
/* --- string forms: quoted, raw (single + double hash) ------------------ */
static MunitResult
test_strings_quoted_raw(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"hello\" #\"raw\"# ##\"raw2\"##");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_STRING, 1, 1, "\"hello\"");
assert_tok(&cap, 1, ST_TOK_RAW_STRING, 1, 9, "#\"raw\"#");
assert_tok(&cap, 2, ST_TOK_RAW_STRING, 1, 17, "##\"raw2\"##");
assert_tok(&cap, 3, ST_TOK_EOF, 1, 27, "");
return MUNIT_OK;
}
/* --- multi-line string (delimiters only; dedent rules are todo 8) ------ */
static MunitResult
test_multiline_string(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"\"\"multi\"\"\"");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"multi\"\"\"");
assert_tok(&cap, 1, ST_TOK_EOF, 1, 12, "");
return MUNIT_OK;
}
/* --- comments ---------------------------------------------------------- */
static MunitResult
test_line_comment(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("a // comment\nb");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
assert_tok(&cap, 1, ST_TOK_LINE_COMMENT, 1, 3, "// comment");
assert_tok(&cap, 2, ST_TOK_IDENT, 2, 1, "b");
assert_tok(&cap, 3, ST_TOK_EOF, 2, 2, "");
return MUNIT_OK;
}
static MunitResult
test_block_comment_nested(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("a /* x /* n */ */ b");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
assert_tok(&cap, 1, ST_TOK_BLOCK_COMMENT, 1, 3, "/* x /* n */ */");
assert_tok(&cap, 2, ST_TOK_IDENT, 1, 19, "b");
assert_tok(&cap, 3, ST_TOK_EOF, 1, 20, "");
return MUNIT_OK;
}
static MunitResult
test_slashdash(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("/- node");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_SLASHDASH, 1, 1, "/-");
assert_tok(&cap, 1, ST_TOK_IDENT, 1, 4, "node");
assert_tok(&cap, 2, ST_TOK_EOF, 1, 8, "");
return MUNIT_OK;
}
/* --- structural tokens -------------------------------------------------- */
static MunitResult
test_semicolons_braces(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("a;b {c}");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
assert_tok(&cap, 1, ST_TOK_SEMICOLON, 1, 2, ";");
assert_tok(&cap, 2, ST_TOK_IDENT, 1, 3, "b");
assert_tok(&cap, 3, ST_TOK_LBRACE, 1, 5, "{");
assert_tok(&cap, 4, ST_TOK_IDENT, 1, 6, "c");
assert_tok(&cap, 5, ST_TOK_RBRACE, 1, 7, "}");
assert_tok(&cap, 6, ST_TOK_EOF, 1, 8, "");
return MUNIT_OK;
}
/* --- identifiers: sign / dotted / hyphen forms ------------------------- */
static MunitResult
test_ident_specials(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("--this .md foo-bar a+b _under");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "--this");
assert_tok(&cap, 1, ST_TOK_IDENT, 1, 8, ".md");
assert_tok(&cap, 2, ST_TOK_IDENT, 1, 12, "foo-bar");
assert_tok(&cap, 3, ST_TOK_IDENT, 1, 20, "a+b");
assert_tok(&cap, 4, ST_TOK_IDENT, 1, 24, "_under");
assert_tok(&cap, 5, ST_TOK_EOF, 1, 30, "");
return MUNIT_OK;
}
/* --- bare keyword identifiers are syntax errors ------------------------ */
static MunitResult
test_bare_keywords_error(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
static const char *const bare[] = {
"inf", "-inf", "nan", "true", "false", "null",
};
size_t i;
for (i = 0; i < sizeof(bare) / sizeof(bare[0]); i++) {
struct capture cap = capture(bare[i]);
assert_error(&cap, 1, 1);
}
/* also rejected in position, with the span on the offending ident */
{
struct capture cap = capture("x inf");
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "x");
assert_error(&cap, 1, 3);
}
return MUNIT_OK;
}
/* --- unterminated strings (all three forms) ---------------------------- */
static MunitResult
test_unterminated_quoted(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("node \"unterminated");
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node");
assert_error(&cap, 1, 6); /* span at the opening quote */
return MUNIT_OK;
}
static MunitResult
test_unterminated_raw(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("#\"unterminated");
assert_error(&cap, 1, 1); /* span at the opening '#' */
return MUNIT_OK;
}
static MunitResult
test_unterminated_multiline(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"\"\"multi");
assert_error(&cap, 1, 1); /* span at the opening '\"\"\"' */
return MUNIT_OK;
}
/* --- malformed numbers -------------------------------------------------- */
static MunitResult
test_bad_hex(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0xZZ");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
static MunitResult
test_hex_no_digits(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0x");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
static MunitResult
test_bad_octal_binary(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0o8");
assert_error(&cap, 1, 1);
cap = capture("0b2");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- leading-dot numbers are errors ------------------------------------ */
static MunitResult
test_dot_number_error(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture(".5");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- invalid escape ----------------------------------------------------- */
static MunitResult
test_invalid_escape(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"\\q\"");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- stray '#' ---------------------------------------------------------- */
static MunitResult
test_hash_alone(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("#");
assert_error(&cap, 1, 1);
cap = capture("##foo");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- bad type annotation ------------------------------------------------ */
static MunitResult
test_bad_type_annotation(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("(u8");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- a stray '}' is a token, not a lexer error (nesting is todo 7) ----- */
static MunitResult
test_stray_brace_is_token(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("}");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_RBRACE, 1, 1, "}");
assert_tok(&cap, 1, ST_TOK_EOF, 1, 2, "");
return MUNIT_OK;
}
/* --- hostile span: column well past 60 stays exact --------------------- */
static MunitResult
test_long_line_span(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
char src[256];
memset(src, ' ', 70);
memcpy(src + 70, "foo", 4);
{
struct capture cap = capture(src);
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 71, "foo");
assert_tok(&cap, 1, ST_TOK_EOF, 1, 74, "");
}
return MUNIT_OK;
}
/* --- line/col tracking across newlines --------------------------------- */
static MunitResult
test_line_col_tracking(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("foo\n bar");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "foo");
assert_tok(&cap, 1, ST_TOK_IDENT, 2, 3, "bar");
assert_tok(&cap, 2, ST_TOK_EOF, 2, 6, "");
/* a multi-line string advances line/col for following tokens */
cap = capture("\"\"\"\nx\n\"\"\" tail");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"\nx\n\"\"\"");
assert_tok(&cap, 1, ST_TOK_IDENT, 3, 5, "tail");
assert_tok(&cap, 2, ST_TOK_EOF, 3, 9, "");
return MUNIT_OK;
}
/* --- token kind names exist and are distinct --------------------------- */
static MunitResult
test_kind_names(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
const char *prev = NULL;
int k;
for (k = ST_TOK_EOF; k <= ST_TOK_SLASHDASH; k++) {
const char *n = st_token_kind_name((enum st_token_kind)k);
munit_assert_not_null(n);
munit_assert_int((int)strlen(n), >, 0);
if (prev != NULL) {
munit_assert_string_not_equal(n, prev);
}
prev = n;
}
return MUNIT_OK;
}
static MunitTest tests[] = {
{ "/lexer/acceptance-node", test_acceptance_node, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/numbers", test_numbers, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/signed-radix-numbers", test_signed_and_radix_numbers, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/keywords", test_keywords, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/type-annotation", test_type_annotation, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/strings-quoted-raw", test_strings_quoted_raw, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/multiline-string", test_multiline_string, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/line-comment", test_line_comment, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/block-comment-nested", test_block_comment_nested, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/slashdash", test_slashdash, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/semicolons-braces", test_semicolons_braces, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/ident-specials", test_ident_specials, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bare-keywords-error", test_bare_keywords_error, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/unterminated-quoted", test_unterminated_quoted, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/unterminated-raw", test_unterminated_raw, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/unterminated-multiline", test_unterminated_multiline, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bad-hex", test_bad_hex, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/hex-no-digits", test_hex_no_digits, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bad-octal-binary", test_bad_octal_binary, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/dot-number-error", test_dot_number_error, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/invalid-escape", test_invalid_escape, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/hash-alone", test_hash_alone, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bad-type-annotation", test_bad_type_annotation, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/stray-brace-is-token", test_stray_brace_is_token, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/long-line-span", test_long_line_span, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/line-col-tracking", test_line_col_tracking, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/kind-names", test_kind_names, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
};
static const MunitSuite suite = {
"/lexer", tests, NULL, 1, MUNIT_SUITE_OPTION_NONE,
};
int
main(int argc, char *argv[MUNIT_ARRAY_PARAM(argc + 1)])
{
return munit_suite_main(&suite, NULL, argc, argv);
}