stupidtools/tests/unit/test_lexer.c

/* LINK: ../../src/kdl/lexer.c ../../src/error.c ../../src/span.c */
/* tests/unit/test_lexer.c
 *
 * Unit tests for the KDL 2.0.0 tokenizer (todo 6).
 *
 * The magic LINK comment on line 1 is REQUIRED by tests/run.sh: it lists
 * the extra .c sources to compile into this test binary (paths relative
 * to tests/unit/, space-separated). lexer.c depends on error.c (typed
 * errors) and span.c (caret rendering), so all three are linked.
 *
 * These tests assert on REAL token streams — kind, span (line/col) and
 * the text slice — never just "does not crash".
 */
#include "munit.h"

#include "error.h"
#include "kdl/lexer.h"

#include <stdbool.h>
#include <stdio.h>
#include <string.h>

#define MAX_TOKS 256
#define MAX_TEXT 512

/* A single captured token. `text` is a NUL-terminated copy of the lexer's
 * text slice (which is otherwise not NUL-terminated). */
struct captured_tok {
    enum st_token_kind kind;
    size_t line;
    size_t col;
    char text[MAX_TEXT];
};

/* A full captured stream. When the lexer hits an error, `had_error` is
 * set and the error's category + span + message are copied out (values,
 * not borrowed pointers) before the lexer is freed — this sidesteps the
 * borrowed-span lifetime entirely inside the test. */
struct capture {
    struct captured_tok toks[MAX_TOKS];
    size_t n;
    bool had_error;
    int err_category;
    size_t err_line;
    size_t err_col;
    char err_msg[MAX_TEXT];
};

/* Tokenize `src` to completion. On a lex error the tokens produced before
 * the error are still captured (and the error info is recorded). */
static struct capture
capture(const char *src)
{
    struct capture cap;
    struct st_lexer *lx;
    struct st_error *err = NULL;

    memset(&cap, 0, sizeof cap);
    lx = st_lexer_new(src, "test.kdl");
    munit_assert_not_null(lx);

    for (;;) {
        struct st_token t;
        enum st_token_kind k = st_lexer_next(lx, &t, &err);

        if (k == ST_TOK_ERROR) {
            cap.had_error = true;
            if (err != NULL) {
                cap.err_category = (int)st_error_category_of(err);
                if (err->span != NULL) {
                    cap.err_line = err->span->line;
                    cap.err_col = err->span->col;
                }
                snprintf(cap.err_msg, sizeof cap.err_msg, "%s",
                         st_error_message(err));
                st_error_free(err);
            }
            break;
        }

        munit_assert_size(cap.n, <, MAX_TOKS);
        cap.toks[cap.n].kind = k;
        cap.toks[cap.n].line = t.span.line;
        cap.toks[cap.n].col = t.span.col;
        {
            size_t len = t.len < MAX_TEXT - 1 ? t.len : MAX_TEXT - 1;
            memcpy(cap.toks[cap.n].text, t.text, len);
            cap.toks[cap.n].text[len] = '\0';
        }
        cap.n++;

        if (k == ST_TOK_EOF) {
            break;
        }
    }

    st_lexer_free(lx);
    return cap;
}

/* Assert token i is (kind, line, col, text). */
static void
assert_tok(const struct capture *cap, size_t i, enum st_token_kind kind,
           size_t line, size_t col, const char *text)
{
    munit_assert_size(i, <, cap->n);
    munit_assert_int(cap->toks[i].kind, ==, kind);
    munit_assert_size(cap->toks[i].line, ==, line);
    munit_assert_size(cap->toks[i].col, ==, col);
    munit_assert_string_equal(cap->toks[i].text, text);
}

/* Assert a lex error occurred with the given (line, col) span. */
static void
assert_error(const struct capture *cap, size_t line, size_t col)
{
    munit_assert_true(cap->had_error);
    munit_assert_int(cap->err_category, ==, ST_ERR_KDL_PARSE);
    munit_assert_size(cap->err_line, ==, line);
    munit_assert_size(cap->err_col, ==, col);
}

/* --- acceptance: the plan's canonical node ----------------------------- */
static MunitResult
test_acceptance_node(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("node 1 2 \"x\" key=\"v\" { child }");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node");
    assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "1");
    assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 8, "2");
    assert_tok(&cap, 3, ST_TOK_STRING, 1, 10, "\"x\"");
    assert_tok(&cap, 4, ST_TOK_IDENT, 1, 14, "key");
    assert_tok(&cap, 5, ST_TOK_EQUALS, 1, 17, "=");
    assert_tok(&cap, 6, ST_TOK_STRING, 1, 18, "\"v\"");
    assert_tok(&cap, 7, ST_TOK_LBRACE, 1, 22, "{");
    assert_tok(&cap, 8, ST_TOK_IDENT, 1, 24, "child");
    assert_tok(&cap, 9, ST_TOK_RBRACE, 1, 30, "}");
    assert_tok(&cap, 10, ST_TOK_EOF, 1, 31, "");
    munit_assert_size(cap.n, ==, 11);
    return MUNIT_OK;
}

/* --- numbers: every radix + sign/float/exponent/underscore ------------ */
static MunitResult
test_numbers(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("0x10 0o17 0b101 123 3.14 1e10 1_000");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "0x10");
    assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 6, "0o17");
    assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 11, "0b101");
    assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 17, "123");
    assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 21, "3.14");
    assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 26, "1e10");
    assert_tok(&cap, 6, ST_TOK_NUMBER, 1, 31, "1_000");
    assert_tok(&cap, 7, ST_TOK_EOF, 1, 36, "");
    return MUNIT_OK;
}

static MunitResult
test_signed_and_radix_numbers(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("-5 +7 -0x10 0XFF 0xff 1e-3");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_NUMBER, 1, 1, "-5");
    assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 4, "+7");
    assert_tok(&cap, 2, ST_TOK_NUMBER, 1, 7, "-0x10");
    assert_tok(&cap, 3, ST_TOK_NUMBER, 1, 13, "0XFF");
    assert_tok(&cap, 4, ST_TOK_NUMBER, 1, 18, "0xff");
    assert_tok(&cap, 5, ST_TOK_NUMBER, 1, 23, "1e-3");
    assert_tok(&cap, 6, ST_TOK_EOF, 1, 27, "");
    return MUNIT_OK;
}

/* --- keyword numbers, booleans, null ---------------------------------- */
static MunitResult
test_keywords(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("#inf #-inf #nan #true #false #null");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_INF, 1, 1, "#inf");
    assert_tok(&cap, 1, ST_TOK_NEG_INF, 1, 6, "#-inf");
    assert_tok(&cap, 2, ST_TOK_NAN, 1, 12, "#nan");
    assert_tok(&cap, 3, ST_TOK_TRUE, 1, 17, "#true");
    assert_tok(&cap, 4, ST_TOK_FALSE, 1, 23, "#false");
    assert_tok(&cap, 5, ST_TOK_NULL, 1, 30, "#null");
    assert_tok(&cap, 6, ST_TOK_EOF, 1, 35, "");
    return MUNIT_OK;
}

/* --- type annotations -------------------------------------------------- */
static MunitResult
test_type_annotation(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("(u8)42 (published)date");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_TYPE, 1, 1, "u8");
    assert_tok(&cap, 1, ST_TOK_NUMBER, 1, 5, "42");
    assert_tok(&cap, 2, ST_TOK_TYPE, 1, 8, "published");
    assert_tok(&cap, 3, ST_TOK_IDENT, 1, 19, "date");
    assert_tok(&cap, 4, ST_TOK_EOF, 1, 23, "");
    return MUNIT_OK;
}

/* --- string forms: quoted, raw (single + double hash) ------------------ */
static MunitResult
test_strings_quoted_raw(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("\"hello\" #\"raw\"# ##\"raw2\"##");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_STRING, 1, 1, "\"hello\"");
    assert_tok(&cap, 1, ST_TOK_RAW_STRING, 1, 9, "#\"raw\"#");
    assert_tok(&cap, 2, ST_TOK_RAW_STRING, 1, 17, "##\"raw2\"##");
    assert_tok(&cap, 3, ST_TOK_EOF, 1, 27, "");
    return MUNIT_OK;
}

/* --- multi-line string (delimiters only; dedent rules are todo 8) ------ */
static MunitResult
test_multiline_string(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("\"\"\"multi\"\"\"");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"multi\"\"\"");
    assert_tok(&cap, 1, ST_TOK_EOF, 1, 12, "");
    return MUNIT_OK;
}

/* --- comments ---------------------------------------------------------- */
static MunitResult
test_line_comment(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("a // comment\nb");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
    assert_tok(&cap, 1, ST_TOK_LINE_COMMENT, 1, 3, "// comment");
    assert_tok(&cap, 2, ST_TOK_IDENT, 2, 1, "b");
    assert_tok(&cap, 3, ST_TOK_EOF, 2, 2, "");
    return MUNIT_OK;
}

static MunitResult
test_block_comment_nested(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("a /* x /* n */ */ b");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
    assert_tok(&cap, 1, ST_TOK_BLOCK_COMMENT, 1, 3, "/* x /* n */ */");
    assert_tok(&cap, 2, ST_TOK_IDENT, 1, 19, "b");
    assert_tok(&cap, 3, ST_TOK_EOF, 1, 20, "");
    return MUNIT_OK;
}

static MunitResult
test_slashdash(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("/- node");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_SLASHDASH, 1, 1, "/-");
    assert_tok(&cap, 1, ST_TOK_IDENT, 1, 4, "node");
    assert_tok(&cap, 2, ST_TOK_EOF, 1, 8, "");
    return MUNIT_OK;
}

/* --- structural tokens -------------------------------------------------- */
static MunitResult
test_semicolons_braces(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("a;b {c}");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "a");
    assert_tok(&cap, 1, ST_TOK_SEMICOLON, 1, 2, ";");
    assert_tok(&cap, 2, ST_TOK_IDENT, 1, 3, "b");
    assert_tok(&cap, 3, ST_TOK_LBRACE, 1, 5, "{");
    assert_tok(&cap, 4, ST_TOK_IDENT, 1, 6, "c");
    assert_tok(&cap, 5, ST_TOK_RBRACE, 1, 7, "}");
    assert_tok(&cap, 6, ST_TOK_EOF, 1, 8, "");
    return MUNIT_OK;
}

/* --- identifiers: sign / dotted / hyphen forms ------------------------- */
static MunitResult
test_ident_specials(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("--this .md foo-bar a+b _under");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "--this");
    assert_tok(&cap, 1, ST_TOK_IDENT, 1, 8, ".md");
    assert_tok(&cap, 2, ST_TOK_IDENT, 1, 12, "foo-bar");
    assert_tok(&cap, 3, ST_TOK_IDENT, 1, 20, "a+b");
    assert_tok(&cap, 4, ST_TOK_IDENT, 1, 24, "_under");
    assert_tok(&cap, 5, ST_TOK_EOF, 1, 30, "");
    return MUNIT_OK;
}

/* --- bare keyword identifiers are syntax errors ------------------------ */
static MunitResult
test_bare_keywords_error(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    static const char *const bare[] = {
        "inf", "-inf", "nan", "true", "false", "null",
    };
    size_t i;

    for (i = 0; i < sizeof(bare) / sizeof(bare[0]); i++) {
        struct capture cap = capture(bare[i]);
        assert_error(&cap, 1, 1);
    }

    /* also rejected in position, with the span on the offending ident */
    {
        struct capture cap = capture("x inf");
        assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "x");
        assert_error(&cap, 1, 3);
    }
    return MUNIT_OK;
}

/* --- unterminated strings (all three forms) ---------------------------- */
static MunitResult
test_unterminated_quoted(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("node \"unterminated");

    assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "node");
    assert_error(&cap, 1, 6); /* span at the opening quote */
    return MUNIT_OK;
}

static MunitResult
test_unterminated_raw(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("#\"unterminated");

    assert_error(&cap, 1, 1); /* span at the opening '#' */
    return MUNIT_OK;
}

static MunitResult
test_unterminated_multiline(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("\"\"\"multi");

    assert_error(&cap, 1, 1); /* span at the opening '\"\"\"' */
    return MUNIT_OK;
}

/* --- malformed numbers -------------------------------------------------- */
static MunitResult
test_bad_hex(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("0xZZ");

    assert_error(&cap, 1, 1);
    return MUNIT_OK;
}

static MunitResult
test_hex_no_digits(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("0x");

    assert_error(&cap, 1, 1);
    return MUNIT_OK;
}

static MunitResult
test_bad_octal_binary(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("0o8");

    assert_error(&cap, 1, 1);

    cap = capture("0b2");
    assert_error(&cap, 1, 1);
    return MUNIT_OK;
}

/* --- leading-dot numbers are errors ------------------------------------ */
static MunitResult
test_dot_number_error(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture(".5");

    assert_error(&cap, 1, 1);
    return MUNIT_OK;
}

/* --- invalid escape ----------------------------------------------------- */
static MunitResult
test_invalid_escape(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("\"\\q\"");

    assert_error(&cap, 1, 1);
    return MUNIT_OK;
}

/* --- stray '#' ---------------------------------------------------------- */
static MunitResult
test_hash_alone(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("#");

    assert_error(&cap, 1, 1);

    cap = capture("##foo");
    assert_error(&cap, 1, 1);
    return MUNIT_OK;
}

/* --- bad type annotation ------------------------------------------------ */
static MunitResult
test_bad_type_annotation(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("(u8");

    assert_error(&cap, 1, 1);
    return MUNIT_OK;
}

/* --- a stray '}' is a token, not a lexer error (nesting is todo 7) ----- */
static MunitResult
test_stray_brace_is_token(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("}");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_RBRACE, 1, 1, "}");
    assert_tok(&cap, 1, ST_TOK_EOF, 1, 2, "");
    return MUNIT_OK;
}

/* --- hostile span: column well past 60 stays exact --------------------- */
static MunitResult
test_long_line_span(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    char src[256];

    memset(src, ' ', 70);
    memcpy(src + 70, "foo", 4);

    {
        struct capture cap = capture(src);
        munit_assert_false(cap.had_error);
        assert_tok(&cap, 0, ST_TOK_IDENT, 1, 71, "foo");
        assert_tok(&cap, 1, ST_TOK_EOF, 1, 74, "");
    }
    return MUNIT_OK;
}

/* --- line/col tracking across newlines --------------------------------- */
static MunitResult
test_line_col_tracking(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    struct capture cap = capture("foo\n  bar");

    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_IDENT, 1, 1, "foo");
    assert_tok(&cap, 1, ST_TOK_IDENT, 2, 3, "bar");
    assert_tok(&cap, 2, ST_TOK_EOF, 2, 6, "");

    /* a multi-line string advances line/col for following tokens */
    cap = capture("\"\"\"\nx\n\"\"\" tail");
    munit_assert_false(cap.had_error);
    assert_tok(&cap, 0, ST_TOK_MULTILINE_STRING, 1, 1, "\"\"\"\nx\n\"\"\"");
    assert_tok(&cap, 1, ST_TOK_IDENT, 3, 5, "tail");
    assert_tok(&cap, 2, ST_TOK_EOF, 3, 9, "");
    return MUNIT_OK;
}

/* --- token kind names exist and are distinct --------------------------- */
static MunitResult
test_kind_names(const MunitParameter params[], void *data)
{
    (void)params;
    (void)data;
    const char *prev = NULL;
    int k;

    for (k = ST_TOK_EOF; k <= ST_TOK_SLASHDASH; k++) {
        const char *n = st_token_kind_name((enum st_token_kind)k);
        munit_assert_not_null(n);
        munit_assert_int((int)strlen(n), >, 0);
        if (prev != NULL) {
            munit_assert_string_not_equal(n, prev);
        }
        prev = n;
    }
    return MUNIT_OK;
}

static MunitTest tests[] = {
    { "/lexer/acceptance-node", test_acceptance_node, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/numbers", test_numbers, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/signed-radix-numbers", test_signed_and_radix_numbers, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/keywords", test_keywords, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/type-annotation", test_type_annotation, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/strings-quoted-raw", test_strings_quoted_raw, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/multiline-string", test_multiline_string, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/line-comment", test_line_comment, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/block-comment-nested", test_block_comment_nested, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/slashdash", test_slashdash, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/semicolons-braces", test_semicolons_braces, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/ident-specials", test_ident_specials, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/bare-keywords-error", test_bare_keywords_error, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/unterminated-quoted", test_unterminated_quoted, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/unterminated-raw", test_unterminated_raw, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/unterminated-multiline", test_unterminated_multiline, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/bad-hex", test_bad_hex, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/hex-no-digits", test_hex_no_digits, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/bad-octal-binary", test_bad_octal_binary, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/dot-number-error", test_dot_number_error, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/invalid-escape", test_invalid_escape, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/hash-alone", test_hash_alone, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/bad-type-annotation", test_bad_type_annotation, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/stray-brace-is-token", test_stray_brace_is_token, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/long-line-span", test_long_line_span, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/line-col-tracking", test_line_col_tracking, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { "/lexer/kind-names", test_kind_names, NULL, NULL,
      MUNIT_TEST_OPTION_NONE, NULL },
    { NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
};

static const MunitSuite suite = {
    "/lexer", tests, NULL, 1, MUNIT_SUITE_OPTION_NONE,
};

int
main(int argc, char *argv[MUNIT_ARRAY_PARAM(argc + 1)])
{
    return munit_suite_main(&suite, NULL, argc, argv);
}