huntedbytheirs/stupidtools
Template
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(kdl): add KDL lexer with source spans

Browse Source
This commit is contained in:
huntedbytheirs
2026-08-28 20:28:57 -04:00
parent 468aa1c4ef
commit 50afaa17ca
3 changed files with 1686 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_lexer.c
+624
View File
@@ -0,0 +1,624 @@
/* LINK: ../../src/kdl/lexer.c ../../src/error.c ../../src/span.c */
/* tests/unit/test_lexer.c
*
* Unit tests for the KDL 2.0.0 tokenizer (todo 6).
*
* The magic LINK comment on line 1 is REQUIRED by tests/run.sh: it lists
* the extra .c sources to compile into this test binary (paths relative
* to tests/unit/, space-separated). lexer.c depends on error.c (typed
* errors) and span.c (caret rendering), so all three are linked.
*
* These tests assert on REAL token streams — kind, span (line/col) and
* the text slice — never just "does not crash".
*/
#include "munit.h"
#include "error.h"
#include "kdl/lexer.h"
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#define MAX_TOKS 256
#define MAX_TEXT 512
/* A single captured token. `text` is a NUL-terminated copy of the lexer's
* text slice (which is otherwise not NUL-terminated). */
struct captured_tok {
enum st_token_kind kind;
size_t line;
size_t col;
char text[MAX_TEXT];
};
/* A full captured stream. When the lexer hits an error, `had_error` is
* set and the error's category + span + message are copied out (values,
* not borrowed pointers) before the lexer is freed — this sidesteps the
* borrowed-span lifetime entirely inside the test. */
struct capture {
struct captured_tok toks[MAX_TOKS];
size_t n;
bool had_error;
int err_category;
size_t err_line;
size_t err_col;
char err_msg[MAX_TEXT];
};
/* Tokenize `src` to completion. On a lex error the tokens produced before
* the error are still captured (and the error info is recorded). */
static struct capture
capture(const char *src)
{
struct capture cap;
struct st_lexer *lx;
struct st_error *err = NULL;
memset(&cap, 0, sizeof cap);
lx = st_lexer_new(src, "test.kdl");
munit_assert_not_null(lx);
for (;;) {
struct st_token t;
enum st_token_kind k = st_lexer_next(lx, &t, &err);
if (k == ST_TOK_ERROR) {
cap.had_error = true;
if (err != NULL) {
cap.err_category = (int)st_error_category_of(err);
if (err->span != NULL) {
cap.err_line = err->span->line;
cap.err_col = err->span->col;
}
snprintf(cap.err_msg, sizeof cap.err_msg, "%s",
st_error_message(err));
st_error_free(err);
}
break;
}
munit_assert_size(cap.n, <, MAX_TOKS);
cap.toks[cap.n].kind = k;
cap.toks[cap.n].line = t.span.line;
cap.toks[cap.n].col = t.span.col;
{
size_t len = t.len < MAX_TEXT - 1 ? t.len : MAX_TEXT - 1;
memcpy(cap.toks[cap.n].text, t.text, len);
cap.toks[cap.n].text[len] = '\0';
}
cap.n++;
if (k == ST_TOK_EOF) {
break;
}
}
st_lexer_free(lx);
return cap;
}
/* Assert token i is (kind, line, col, text). */
static void
assert_tok(const struct capture *cap, size_t i, enum st_token_kind kind,
size_t line, size_t col, const char *text)
{
munit_assert_size(i, <, cap->n);
munit_assert_int(cap->toks[i].kind, ==, kind);
munit_assert_size(cap->toks[i].line, ==, line);
munit_assert_size(cap->toks[i].col, ==, col);
munit_assert_string_equal(cap->toks[i].text, text);
}
/* Assert a lex error occurred with the given (line, col) span. */
static void
assert_error(const struct capture *cap, size_t line, size_t col)
{
munit_assert_true(cap->had_error);
munit_assert_int(cap->err_category, ==, ST_ERR_KDL_PARSE);
munit_assert_size(cap->err_line, ==, line);
munit_assert_size(cap->err_col, ==, col);
}
/* --- acceptance: the plan's canonical node ----------------------------- */
static MunitResult
test_acceptance_node(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("node 1 2 \"x\" key=\"v\" { child }");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "1");
assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 8, "2");
assert_tok(&cap, 3, ST_TOK_STRING, 1, 10, "\"x\"");
assert_tok(&cap, 4, ST_TOK_IDENT, 1, 14, "key");
assert_tok(&cap, 5, ST_TOK_EQUALS, 1, 17, "=");
assert_tok(&cap, 6, ST_TOK_STRING, 1, 18, "\"v\"");
assert_tok(&cap, 7, ST_TOK_LBRACE, 1, 22, "{");
assert_tok(&cap, 8, ST_TOK_IDENT, 1, 24, "child");
assert_tok(&cap, 9, ST_TOK_RBRACE, 1, 30, "}");
assert_tok(&cap, 10, ST_TOK_EOF, 1, 31, "");
munit_assert_size(cap.n, ==, 11);
return MUNIT_OK;
}
/* --- numbers: every radix + sign/float/exponent/underscore ------------ */
static MunitResult
test_numbers(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0x10 0o17 0b101 123 3.14 1e10 1_000");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "0x10");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "0o17");
assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 11, "0b101");
assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 17, "123");
assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 21, "3.14");
assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 26, "1e10");
assert_tok(&cap, 6, ST_TOK_NUMBER, 1, 31, "1_000");
assert_tok(&cap, 7, ST_TOK_EOF, 1, 36, "");
return MUNIT_OK;
}
static MunitResult
test_signed_and_radix_numbers(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("-5 +7 -0x10 0XFF 0xff 1e-3");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "-5");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 4, "+7");
assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 7, "-0x10");
assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 13, "0XFF");
assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 18, "0xff");
assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 23, "1e-3");
assert_tok(&cap, 6, ST_TOK_EOF, 1, 27, "");
return MUNIT_OK;
}
/* --- keyword numbers, booleans, null ---------------------------------- */
static MunitResult
test_keywords(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("#inf #-inf #nan #true #false #null");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_INF, 1, 1, "#inf");
assert_tok(&cap, 1, ST_TOK_NEG_INF, 1, 6, "#-inf");
assert_tok(&cap, 2, ST_TOK_NAN, 1, 12, "#nan");
assert_tok(&cap, 3, ST_TOK_TRUE, 1, 17, "#true");
assert_tok(&cap, 4, ST_TOK_FALSE, 1, 23, "#false");
assert_tok(&cap, 5, ST_TOK_NULL, 1, 30, "#null");
assert_tok(&cap, 6, ST_TOK_EOF, 1, 35, "");
return MUNIT_OK;
}
/* --- type annotations -------------------------------------------------- */
static MunitResult
test_type_annotation(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("(u8)42 (published)date");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_TYPE, 1, 1, "u8");
assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 5, "42");
assert_tok(&cap, 2, ST_TOK_TYPE, 1, 8, "published");
assert_tok(&cap, 3, ST_TOK_IDENT, 1, 19, "date");
assert_tok(&cap, 4, ST_TOK_EOF, 1, 23, "");
return MUNIT_OK;
}
/* --- string forms: quoted, raw (single + double hash) ------------------ */
static MunitResult
test_strings_quoted_raw(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"hello\" #\"raw\"# ##\"raw2\"##");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_STRING, 1, 1, "\"hello\"");
assert_tok(&cap, 1, ST_TOK_RAW_STRING, 1, 9, "#\"raw\"#");
assert_tok(&cap, 2, ST_TOK_RAW_STRING, 1, 17, "##\"raw2\"##");
assert_tok(&cap, 3, ST_TOK_EOF, 1, 27, "");
return MUNIT_OK;
}
/* --- multi-line string (delimiters only; dedent rules are todo 8) ------ */
static MunitResult
test_multiline_string(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"\"\"multi\"\"\"");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"multi\"\"\"");
assert_tok(&cap, 1, ST_TOK_EOF, 1, 12, "");
return MUNIT_OK;
}
/* --- comments ---------------------------------------------------------- */
static MunitResult
test_line_comment(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("a // comment\nb");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
assert_tok(&cap, 1, ST_TOK_LINE_COMMENT, 1, 3, "// comment");
assert_tok(&cap, 2, ST_TOK_IDENT, 2, 1, "b");
assert_tok(&cap, 3, ST_TOK_EOF, 2, 2, "");
return MUNIT_OK;
}
static MunitResult
test_block_comment_nested(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("a /* x /* n */ */ b");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
assert_tok(&cap, 1, ST_TOK_BLOCK_COMMENT, 1, 3, "/* x /* n */ */");
assert_tok(&cap, 2, ST_TOK_IDENT, 1, 19, "b");
assert_tok(&cap, 3, ST_TOK_EOF, 1, 20, "");
return MUNIT_OK;
}
static MunitResult
test_slashdash(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("/- node");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_SLASHDASH, 1, 1, "/-");
assert_tok(&cap, 1, ST_TOK_IDENT, 1, 4, "node");
assert_tok(&cap, 2, ST_TOK_EOF, 1, 8, "");
return MUNIT_OK;
}
/* --- structural tokens -------------------------------------------------- */
static MunitResult
test_semicolons_braces(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("a;b {c}");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
assert_tok(&cap, 1, ST_TOK_SEMICOLON, 1, 2, ";");
assert_tok(&cap, 2, ST_TOK_IDENT, 1, 3, "b");
assert_tok(&cap, 3, ST_TOK_LBRACE, 1, 5, "{");
assert_tok(&cap, 4, ST_TOK_IDENT, 1, 6, "c");
assert_tok(&cap, 5, ST_TOK_RBRACE, 1, 7, "}");
assert_tok(&cap, 6, ST_TOK_EOF, 1, 8, "");
return MUNIT_OK;
}
/* --- identifiers: sign / dotted / hyphen forms ------------------------- */
static MunitResult
test_ident_specials(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("--this .md foo-bar a+b _under");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "--this");
assert_tok(&cap, 1, ST_TOK_IDENT, 1, 8, ".md");
assert_tok(&cap, 2, ST_TOK_IDENT, 1, 12, "foo-bar");
assert_tok(&cap, 3, ST_TOK_IDENT, 1, 20, "a+b");
assert_tok(&cap, 4, ST_TOK_IDENT, 1, 24, "_under");
assert_tok(&cap, 5, ST_TOK_EOF, 1, 30, "");
return MUNIT_OK;
}
/* --- bare keyword identifiers are syntax errors ------------------------ */
static MunitResult
test_bare_keywords_error(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
static const char *const bare[] = {
"inf", "-inf", "nan", "true", "false", "null",
};
size_t i;
for (i = 0; i < sizeof(bare) / sizeof(bare[0]); i++) {
struct capture cap = capture(bare[i]);
assert_error(&cap, 1, 1);
}
/* also rejected in position, with the span on the offending ident */
{
struct capture cap = capture("x inf");
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "x");
assert_error(&cap, 1, 3);
}
return MUNIT_OK;
}
/* --- unterminated strings (all three forms) ---------------------------- */
static MunitResult
test_unterminated_quoted(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("node \"unterminated");
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node");
assert_error(&cap, 1, 6); /* span at the opening quote */
return MUNIT_OK;
}
static MunitResult
test_unterminated_raw(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("#\"unterminated");
assert_error(&cap, 1, 1); /* span at the opening '#' */
return MUNIT_OK;
}
static MunitResult
test_unterminated_multiline(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"\"\"multi");
assert_error(&cap, 1, 1); /* span at the opening '\"\"\"' */
return MUNIT_OK;
}
/* --- malformed numbers -------------------------------------------------- */
static MunitResult
test_bad_hex(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0xZZ");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
static MunitResult
test_hex_no_digits(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0x");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
static MunitResult
test_bad_octal_binary(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("0o8");
assert_error(&cap, 1, 1);
cap = capture("0b2");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- leading-dot numbers are errors ------------------------------------ */
static MunitResult
test_dot_number_error(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture(".5");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- invalid escape ----------------------------------------------------- */
static MunitResult
test_invalid_escape(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("\"\\q\"");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- stray '#' ---------------------------------------------------------- */
static MunitResult
test_hash_alone(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("#");
assert_error(&cap, 1, 1);
cap = capture("##foo");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- bad type annotation ------------------------------------------------ */
static MunitResult
test_bad_type_annotation(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("(u8");
assert_error(&cap, 1, 1);
return MUNIT_OK;
}
/* --- a stray '}' is a token, not a lexer error (nesting is todo 7) ----- */
static MunitResult
test_stray_brace_is_token(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("}");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_RBRACE, 1, 1, "}");
assert_tok(&cap, 1, ST_TOK_EOF, 1, 2, "");
return MUNIT_OK;
}
/* --- hostile span: column well past 60 stays exact --------------------- */
static MunitResult
test_long_line_span(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
char src[256];
memset(src, ' ', 70);
memcpy(src + 70, "foo", 4);
{
struct capture cap = capture(src);
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 71, "foo");
assert_tok(&cap, 1, ST_TOK_EOF, 1, 74, "");
}
return MUNIT_OK;
}
/* --- line/col tracking across newlines --------------------------------- */
static MunitResult
test_line_col_tracking(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
struct capture cap = capture("foo\n bar");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "foo");
assert_tok(&cap, 1, ST_TOK_IDENT, 2, 3, "bar");
assert_tok(&cap, 2, ST_TOK_EOF, 2, 6, "");
/* a multi-line string advances line/col for following tokens */
cap = capture("\"\"\"\nx\n\"\"\" tail");
munit_assert_false(cap.had_error);
assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"\nx\n\"\"\"");
assert_tok(&cap, 1, ST_TOK_IDENT, 3, 5, "tail");
assert_tok(&cap, 2, ST_TOK_EOF, 3, 9, "");
return MUNIT_OK;
}
/* --- token kind names exist and are distinct --------------------------- */
static MunitResult
test_kind_names(const MunitParameter params[], void *data)
{
(void)params;
(void)data;
const char *prev = NULL;
int k;
for (k = ST_TOK_EOF; k <= ST_TOK_SLASHDASH; k++) {
const char *n = st_token_kind_name((enum st_token_kind)k);
munit_assert_not_null(n);
munit_assert_int((int)strlen(n), >, 0);
if (prev != NULL) {
munit_assert_string_not_equal(n, prev);
}
prev = n;
}
return MUNIT_OK;
}
static MunitTest tests[] = {
{ "/lexer/acceptance-node", test_acceptance_node, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/numbers", test_numbers, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/signed-radix-numbers", test_signed_and_radix_numbers, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/keywords", test_keywords, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/type-annotation", test_type_annotation, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/strings-quoted-raw", test_strings_quoted_raw, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/multiline-string", test_multiline_string, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/line-comment", test_line_comment, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/block-comment-nested", test_block_comment_nested, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/slashdash", test_slashdash, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/semicolons-braces", test_semicolons_braces, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/ident-specials", test_ident_specials, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bare-keywords-error", test_bare_keywords_error, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/unterminated-quoted", test_unterminated_quoted, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/unterminated-raw", test_unterminated_raw, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/unterminated-multiline", test_unterminated_multiline, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bad-hex", test_bad_hex, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/hex-no-digits", test_hex_no_digits, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bad-octal-binary", test_bad_octal_binary, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/dot-number-error", test_dot_number_error, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/invalid-escape", test_invalid_escape, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/hash-alone", test_hash_alone, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/bad-type-annotation", test_bad_type_annotation, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/stray-brace-is-token", test_stray_brace_is_token, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/long-line-span", test_long_line_span, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/line-col-tracking", test_line_col_tracking, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ "/lexer/kind-names", test_kind_names, NULL, NULL,
MUNIT_TEST_OPTION_NONE, NULL },
{ NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
};
static const MunitSuite suite = {
"/lexer", tests, NULL, 1, MUNIT_SUITE_OPTION_NONE,
};
int
main(int argc, char *argv[MUNIT_ARRAY_PARAM(argc + 1)])
{
return munit_suite_main(&suite, NULL, argc, argv);
}