Template
feat(kdl): add KDL lexer with source spans
This commit is contained in:
+943
@@ -0,0 +1,943 @@
|
||||
/*
|
||||
* lexer.c - KDL 2.0.0 tokenizer for stupidtools (todo 6).
|
||||
*
|
||||
* Produces a FLAT token stream with a source span on every token. It does
|
||||
* NOT parse node/argument/property structure (the parser does, todo 7).
|
||||
* See lexer.h for the full API + error-span-lifetime contract.
|
||||
*
|
||||
* Design decisions (mirrored in .omo/notepads/stupidtools/learnings.md):
|
||||
* - identifiers are scanned greedily; non-ASCII bytes are identifier
|
||||
* characters (the non-identifier set is the ASCII one from §3.10.2).
|
||||
* - bare `inf` / `-inf` / `nan` / `true` / `false` / `null` are syntax
|
||||
* errors; the `#`-prefixed keyword forms are distinct token kinds.
|
||||
* - multi-line strings are delimited by `"""` only; the spec's "first
|
||||
* line must be a newline" + dedent-prefix rules are VALUE rules and
|
||||
* are deferred to the value model (todo 8), so `"""multi"""` tokenizes.
|
||||
* - columns are byte-based (not codepoints); full Unicode grapheme
|
||||
* columns and disallowed-literal-code-point validation are deferred.
|
||||
*
|
||||
* Copyright (c) 2026 huntedbytheirs
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
#include "kdl/lexer.h"
|
||||
|
||||
#include "error.h"
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
struct st_lexer {
|
||||
const char *src; /* borrowed source buffer (NUL-terminated) */
|
||||
const char *pos; /* current scan position */
|
||||
const char *filename; /* borrowed; stamped into every span */
|
||||
size_t line; /* current 1-based line */
|
||||
size_t col; /* current 1-based column (bytes) */
|
||||
struct st_span err_span; /* stable span for the most recent error */
|
||||
};
|
||||
|
||||
/*
|
||||
* Character classes. Non-ASCII bytes are deliberately treated as
|
||||
* identifier characters (see header note); the classification below only
|
||||
* special-cases ASCII + the UTF-8 newline sequences (NEL/LS/PS).
|
||||
*/
|
||||
|
||||
static bool
|
||||
is_digit(unsigned char c)
|
||||
{
|
||||
return c >= '0' && c <= '9';
|
||||
}
|
||||
|
||||
static bool
|
||||
is_hex_digit(unsigned char c)
|
||||
{
|
||||
return is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
static bool
|
||||
is_oct_digit(unsigned char c)
|
||||
{
|
||||
return c >= '0' && c <= '7';
|
||||
}
|
||||
|
||||
static bool
|
||||
is_bin_digit(unsigned char c)
|
||||
{
|
||||
return c == '0' || c == '1';
|
||||
}
|
||||
|
||||
/* Non-newline whitespace (ASCII subset; full Unicode White_Space deferred). */
|
||||
static bool
|
||||
is_space(unsigned char c)
|
||||
{
|
||||
return c == ' ' || c == '\t';
|
||||
}
|
||||
|
||||
/* Non-identifier ASCII chars per KDL §3.10.2: \ / ( ) { } ; [ ] " # = */
|
||||
static bool
|
||||
is_nonident(unsigned char c)
|
||||
{
|
||||
switch (c) {
|
||||
case '\\':
|
||||
case '/':
|
||||
case '(':
|
||||
case ')':
|
||||
case '{':
|
||||
case '}':
|
||||
case ';':
|
||||
case '[':
|
||||
case ']':
|
||||
case '"':
|
||||
case '#':
|
||||
case '=':
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/* Newline sequences per KDL §3.18: CRLF (one newline), CR, LF, VT, FF,
|
||||
* NEL (U+0085), LS (U+2028), PS (U+2029). */
|
||||
static bool
|
||||
is_newline_at(const char *p)
|
||||
{
|
||||
unsigned char c = (unsigned char)p[0];
|
||||
|
||||
if (c == '\r' || c == '\n' || c == '\v' || c == '\f') {
|
||||
return true;
|
||||
}
|
||||
if (c == 0xC2 && (unsigned char)p[1] == 0x85) {
|
||||
return true; /* NEL */
|
||||
}
|
||||
if (c == 0xE2 && (unsigned char)p[1] == 0x80 &&
|
||||
((unsigned char)p[2] == 0xA8 || (unsigned char)p[2] == 0xA9)) {
|
||||
return true; /* LS, PS */
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Advance one character, updating line/col. CRLF counts as one newline. */
|
||||
static void
|
||||
advance(struct st_lexer *lx)
|
||||
{
|
||||
unsigned char c = (unsigned char)*lx->pos;
|
||||
|
||||
if (c == '\0') {
|
||||
return;
|
||||
}
|
||||
if (is_newline_at(lx->pos)) {
|
||||
if (c == '\r' && lx->pos[1] == '\n') {
|
||||
lx->pos += 2;
|
||||
} else if (c == 0xC2) {
|
||||
lx->pos += 2;
|
||||
} else if (c == 0xE2) {
|
||||
lx->pos += 3;
|
||||
} else {
|
||||
lx->pos += 1;
|
||||
}
|
||||
lx->line++;
|
||||
lx->col = 1;
|
||||
return;
|
||||
}
|
||||
lx->pos += 1;
|
||||
lx->col += 1;
|
||||
}
|
||||
|
||||
/* True when the cursor is on a byte that may continue an identifier. */
|
||||
static bool
|
||||
at_ident_char(const struct st_lexer *lx)
|
||||
{
|
||||
unsigned char c = (unsigned char)*lx->pos;
|
||||
|
||||
if (c == '\0' || is_nonident(c) || is_space(c)) {
|
||||
return false;
|
||||
}
|
||||
return !is_newline_at(lx->pos);
|
||||
}
|
||||
|
||||
/* True when the cursor is on a byte that may START an identifier (i.e. not
|
||||
* a digit, non-ident char, whitespace, or newline). `+`/`-`/`.` are handled
|
||||
* by the caller because they can also begin numbers. */
|
||||
static bool
|
||||
at_ident_start(const struct st_lexer *lx)
|
||||
{
|
||||
unsigned char c = (unsigned char)*lx->pos;
|
||||
|
||||
if (c == '\0' || is_digit(c) || is_nonident(c) || is_space(c)) {
|
||||
return false;
|
||||
}
|
||||
return !is_newline_at(lx->pos);
|
||||
}
|
||||
|
||||
/* Build a KDL-parse error at `sp` and return ST_TOK_ERROR. The span is
|
||||
* stored in the lexer so the error's borrowed pointer stays valid until
|
||||
* the next st_lexer_next/free (see lexer.h). */
|
||||
static enum st_token_kind
|
||||
fail(struct st_lexer *lx, struct st_span sp, const char *msg,
|
||||
struct st_error **err)
|
||||
{
|
||||
if (err != NULL) {
|
||||
lx->err_span = sp;
|
||||
*err = st_error_at(st_error_kdl_parse(msg), &lx->err_span);
|
||||
}
|
||||
return ST_TOK_ERROR;
|
||||
}
|
||||
|
||||
static void
|
||||
emit(struct st_token *out, enum st_token_kind kind, struct st_span span,
|
||||
const char *text, size_t len)
|
||||
{
|
||||
out->kind = kind;
|
||||
out->span = span;
|
||||
out->text = text;
|
||||
out->len = len;
|
||||
}
|
||||
|
||||
/* Is `s` (len bytes) one of the bare keyword identifiers that KDL forbids
|
||||
* as an identifier string (must be written with `#` or quoted)? */
|
||||
static bool
|
||||
is_bare_keyword(const char *s, size_t len)
|
||||
{
|
||||
static const char *const keywords[] = {
|
||||
"true", "false", "null", "inf", "-inf", "nan",
|
||||
};
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) {
|
||||
size_t klen = strlen(keywords[i]);
|
||||
if (len == klen && memcmp(s, keywords[i], klen) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* --- line continuations ------------------------------------------------ */
|
||||
|
||||
/* Consume an escline: '\' ws* (single-line-comment | newline | eof).
|
||||
* Returns 0 on success, -1 on error (*err set). */
|
||||
static int
|
||||
lex_escline(struct st_lexer *lx, struct st_error **err)
|
||||
{
|
||||
struct st_span sp = { lx->filename, lx->line, lx->col };
|
||||
|
||||
advance(lx); /* '\' */
|
||||
while (*lx->pos != '\0' && is_space((unsigned char)*lx->pos)) {
|
||||
advance(lx);
|
||||
}
|
||||
if (*lx->pos == '\0') {
|
||||
return 0; /* eof is a legal escline terminator */
|
||||
}
|
||||
if (is_newline_at(lx->pos)) {
|
||||
advance(lx);
|
||||
return 0;
|
||||
}
|
||||
if (lx->pos[0] == '/' && lx->pos[1] == '/') {
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
while (*lx->pos != '\0' && !is_newline_at(lx->pos)) {
|
||||
advance(lx);
|
||||
}
|
||||
if (*lx->pos != '\0') {
|
||||
advance(lx); /* the terminating newline */
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
(void)fail(lx, sp,
|
||||
"invalid line continuation: '\\' must be followed by a newline "
|
||||
"or a line comment", err);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* --- identifiers ------------------------------------------------------- */
|
||||
|
||||
static enum st_token_kind
|
||||
lex_ident(struct st_lexer *lx, struct st_token *out, struct st_span start,
|
||||
struct st_error **err)
|
||||
{
|
||||
const char *begin = lx->pos;
|
||||
|
||||
advance(lx); /* first char is a valid ident-start (caller guarantees) */
|
||||
while (at_ident_char(lx)) {
|
||||
advance(lx);
|
||||
}
|
||||
|
||||
if (is_bare_keyword(begin, (size_t)(lx->pos - begin))) {
|
||||
return fail(lx, start,
|
||||
"bare keyword identifier; write it as a '#'-prefixed keyword "
|
||||
"or a quoted string", err);
|
||||
}
|
||||
|
||||
emit(out, ST_TOK_IDENT, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_IDENT;
|
||||
}
|
||||
|
||||
/* --- numbers ----------------------------------------------------------- */
|
||||
|
||||
static enum st_token_kind
|
||||
lex_number(struct st_lexer *lx, struct st_token *out, struct st_span start,
|
||||
struct st_error **err)
|
||||
{
|
||||
const char *begin = lx->pos;
|
||||
|
||||
if (*lx->pos == '+' || *lx->pos == '-') {
|
||||
advance(lx); /* sign */
|
||||
}
|
||||
|
||||
if (lx->pos[0] == '0' && (lx->pos[1] == 'x' || lx->pos[1] == 'X')) {
|
||||
advance(lx);
|
||||
advance(lx); /* 0x */
|
||||
if (!is_hex_digit((unsigned char)*lx->pos)) {
|
||||
return fail(lx, start,
|
||||
"invalid hexadecimal number: expected a hex digit after "
|
||||
"'0x'", err);
|
||||
}
|
||||
while (is_hex_digit((unsigned char)*lx->pos) ||
|
||||
*lx->pos == '_') {
|
||||
advance(lx);
|
||||
}
|
||||
} else if (lx->pos[0] == '0' &&
|
||||
(lx->pos[1] == 'o' || lx->pos[1] == 'O')) {
|
||||
advance(lx);
|
||||
advance(lx); /* 0o */
|
||||
if (!is_oct_digit((unsigned char)*lx->pos)) {
|
||||
return fail(lx, start,
|
||||
"invalid octal number: expected an octal digit after '0o'",
|
||||
err);
|
||||
}
|
||||
while (is_oct_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
|
||||
advance(lx);
|
||||
}
|
||||
} else if (lx->pos[0] == '0' &&
|
||||
(lx->pos[1] == 'b' || lx->pos[1] == 'B')) {
|
||||
advance(lx);
|
||||
advance(lx); /* 0b */
|
||||
if (!is_bin_digit((unsigned char)*lx->pos)) {
|
||||
return fail(lx, start,
|
||||
"invalid binary number: expected a binary digit after '0b'",
|
||||
err);
|
||||
}
|
||||
while (is_bin_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
|
||||
advance(lx);
|
||||
}
|
||||
} else {
|
||||
/* decimal: integer ('.' integer)? exponent? */
|
||||
while (is_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
|
||||
advance(lx);
|
||||
}
|
||||
if (*lx->pos == '.' && is_digit((unsigned char)lx->pos[1])) {
|
||||
advance(lx); /* '.' */
|
||||
while (is_digit((unsigned char)*lx->pos) || *lx->pos == '_') {
|
||||
advance(lx);
|
||||
}
|
||||
}
|
||||
if (*lx->pos == 'e' || *lx->pos == 'E') {
|
||||
const char *save = lx->pos;
|
||||
size_t save_line = lx->line;
|
||||
size_t save_col = lx->col;
|
||||
|
||||
advance(lx); /* e */
|
||||
if (*lx->pos == '+' || *lx->pos == '-') {
|
||||
advance(lx);
|
||||
}
|
||||
if (!is_digit((unsigned char)*lx->pos)) {
|
||||
/* not a real exponent: back off */
|
||||
lx->pos = save;
|
||||
lx->line = save_line;
|
||||
lx->col = save_col;
|
||||
} else {
|
||||
while (is_digit((unsigned char)*lx->pos) ||
|
||||
*lx->pos == '_') {
|
||||
advance(lx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emit(out, ST_TOK_NUMBER, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_NUMBER;
|
||||
}
|
||||
|
||||
/* Dispatch for bytes that may start either a number or an identifier
|
||||
* (digits, '+', '-', '.'). */
|
||||
static enum st_token_kind
|
||||
lex_value_start(struct st_lexer *lx, struct st_token *out,
|
||||
struct st_span start, struct st_error **err)
|
||||
{
|
||||
unsigned char c = (unsigned char)*lx->pos;
|
||||
|
||||
if (is_digit(c)) {
|
||||
return lex_number(lx, out, start, err);
|
||||
}
|
||||
if (c == '+' || c == '-') {
|
||||
unsigned char n = (unsigned char)lx->pos[1];
|
||||
|
||||
if (is_digit(n)) {
|
||||
return lex_number(lx, out, start, err);
|
||||
}
|
||||
if (n == '.' && is_digit((unsigned char)lx->pos[2])) {
|
||||
return fail(lx, start,
|
||||
"number must have a leading digit before the decimal point",
|
||||
err);
|
||||
}
|
||||
return lex_ident(lx, out, start, err);
|
||||
}
|
||||
if (c == '.') {
|
||||
if (is_digit((unsigned char)lx->pos[1])) {
|
||||
return fail(lx, start,
|
||||
"number must have a leading digit (write '0.1', not '.1')",
|
||||
err);
|
||||
}
|
||||
return lex_ident(lx, out, start, err);
|
||||
}
|
||||
return lex_ident(lx, out, start, err);
|
||||
}
|
||||
|
||||
/* --- strings ----------------------------------------------------------- */
|
||||
|
||||
/* Scan one escape sequence (at a '\'); validates it but does not unescape.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
static int
|
||||
lex_escape(struct st_lexer *lx, struct st_span start, struct st_error **err)
|
||||
{
|
||||
unsigned char c;
|
||||
|
||||
advance(lx); /* '\' */
|
||||
c = (unsigned char)*lx->pos;
|
||||
|
||||
if (c == '\0') {
|
||||
(void)fail(lx, start, "unterminated escape sequence in string", err);
|
||||
return -1;
|
||||
}
|
||||
switch (c) {
|
||||
case 'n':
|
||||
case 'r':
|
||||
case 't':
|
||||
case '\\':
|
||||
case '"':
|
||||
case 'b':
|
||||
case 'f':
|
||||
case 's':
|
||||
advance(lx);
|
||||
return 0;
|
||||
case 'u': {
|
||||
int ndigits = 0;
|
||||
|
||||
advance(lx); /* u */
|
||||
if (*lx->pos != '{') {
|
||||
(void)fail(lx, start,
|
||||
"invalid unicode escape: expected '{' after '\\u'", err);
|
||||
return -1;
|
||||
}
|
||||
advance(lx); /* { */
|
||||
while (is_hex_digit((unsigned char)*lx->pos)) {
|
||||
advance(lx);
|
||||
ndigits++;
|
||||
}
|
||||
if (*lx->pos != '}') {
|
||||
(void)fail(lx, start,
|
||||
"invalid unicode escape: expected '}'", err);
|
||||
return -1;
|
||||
}
|
||||
if (ndigits == 0 || ndigits > 6) {
|
||||
(void)fail(lx, start,
|
||||
"invalid unicode escape: expected 1-6 hex digits", err);
|
||||
return -1;
|
||||
}
|
||||
advance(lx); /* } */
|
||||
return 0;
|
||||
}
|
||||
default:
|
||||
/* whitespace escape: '\' + one-or-more whitespace/newlines */
|
||||
if (is_space(c) || is_newline_at(lx->pos)) {
|
||||
while (*lx->pos != '\0' &&
|
||||
(is_space((unsigned char)*lx->pos) ||
|
||||
is_newline_at(lx->pos))) {
|
||||
advance(lx);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
(void)fail(lx, start, "invalid escape sequence in string", err);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Single-line quoted string: "..." (raw disables escapes). */
|
||||
static enum st_token_kind
|
||||
lex_quoted_string(struct st_lexer *lx, struct st_token *out,
|
||||
struct st_span start, struct st_error **err, bool raw)
|
||||
{
|
||||
const char *begin = lx->pos;
|
||||
|
||||
advance(lx); /* opening '"' */
|
||||
|
||||
for (;;) {
|
||||
unsigned char c = (unsigned char)*lx->pos;
|
||||
|
||||
if (c == '\0') {
|
||||
return fail(lx, start, "unterminated string literal", err);
|
||||
}
|
||||
if (c == '"') {
|
||||
advance(lx); /* closing '"' */
|
||||
break;
|
||||
}
|
||||
if (!raw && c == '\\') {
|
||||
if (lex_escape(lx, start, err) < 0) {
|
||||
return ST_TOK_ERROR;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (is_newline_at(lx->pos)) {
|
||||
return fail(lx, start,
|
||||
"unterminated string literal (newline before closing quote)",
|
||||
err);
|
||||
}
|
||||
advance(lx);
|
||||
}
|
||||
|
||||
emit(out, ST_TOK_STRING, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_STRING;
|
||||
}
|
||||
|
||||
/* Multi-line string: """...""" (raw disables escapes). Only the delimiters
|
||||
* are recognized here; dedent/value rules are deferred to todo 8. */
|
||||
static enum st_token_kind
|
||||
lex_multiline_string(struct st_lexer *lx, struct st_token *out,
|
||||
struct st_span start, struct st_error **err, bool raw)
|
||||
{
|
||||
const char *begin = lx->pos;
|
||||
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
advance(lx); /* opening '"""' */
|
||||
|
||||
for (;;) {
|
||||
if (*lx->pos == '\0') {
|
||||
return fail(lx, start, "unterminated multi-line string", err);
|
||||
}
|
||||
if (lx->pos[0] == '"' && lx->pos[1] == '"' && lx->pos[2] == '"') {
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
advance(lx); /* closing '"""' */
|
||||
break;
|
||||
}
|
||||
if (!raw && *lx->pos == '\\') {
|
||||
if (lex_escape(lx, start, err) < 0) {
|
||||
return ST_TOK_ERROR;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
advance(lx);
|
||||
}
|
||||
|
||||
emit(out, ST_TOK_MULTILINE_STRING, start, begin,
|
||||
(size_t)(lx->pos - begin));
|
||||
return ST_TOK_MULTILINE_STRING;
|
||||
}
|
||||
|
||||
/* Raw string: '#'* '"' body '"' '#'* or '#'* '"""' body '"""' '#'*.
|
||||
* The body may contain '"' / '#' as long as it is not the closing
|
||||
* delimiter (the closing quote followed by exactly n '#'). */
|
||||
static enum st_token_kind
|
||||
lex_raw_string(struct st_lexer *lx, struct st_token *out,
|
||||
struct st_span start, struct st_error **err)
|
||||
{
|
||||
const char *begin = lx->pos;
|
||||
size_t n = 0;
|
||||
size_t i;
|
||||
bool multi;
|
||||
|
||||
while (*lx->pos == '#') {
|
||||
advance(lx);
|
||||
n++;
|
||||
}
|
||||
|
||||
multi = (lx->pos[0] == '"' && lx->pos[1] == '"' && lx->pos[2] == '"');
|
||||
|
||||
if (multi) {
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
advance(lx); /* '"""' */
|
||||
for (;;) {
|
||||
if (*lx->pos == '\0') {
|
||||
return fail(lx, start,
|
||||
"unterminated raw multi-line string", err);
|
||||
}
|
||||
if (lx->pos[0] == '"' && lx->pos[1] == '"' &&
|
||||
lx->pos[2] == '"') {
|
||||
bool closes = true;
|
||||
|
||||
for (i = 0; i < n; i++) {
|
||||
if (lx->pos[3 + i] != '#') {
|
||||
closes = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (closes) {
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
for (i = 0; i < n; i++) {
|
||||
advance(lx);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
advance(lx);
|
||||
}
|
||||
} else {
|
||||
advance(lx); /* '"' */
|
||||
for (;;) {
|
||||
if (*lx->pos == '\0') {
|
||||
return fail(lx, start,
|
||||
"unterminated raw string literal", err);
|
||||
}
|
||||
if (lx->pos[0] == '"') {
|
||||
bool closes = true;
|
||||
|
||||
for (i = 0; i < n; i++) {
|
||||
if (lx->pos[1 + i] != '#') {
|
||||
closes = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (closes) {
|
||||
advance(lx); /* '"' */
|
||||
for (i = 0; i < n; i++) {
|
||||
advance(lx);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (is_newline_at(lx->pos)) {
|
||||
return fail(lx, start,
|
||||
"unterminated raw string literal (newline before "
|
||||
"closing quote)", err);
|
||||
}
|
||||
advance(lx);
|
||||
}
|
||||
}
|
||||
|
||||
emit(out, ST_TOK_RAW_STRING, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_RAW_STRING;
|
||||
}
|
||||
|
||||
/* --- keywords (#inf / #-inf / #nan / #true / #false / #null) ----------- */
|
||||
|
||||
static enum st_token_kind
|
||||
lex_keyword(struct st_lexer *lx, struct st_token *out, struct st_span start,
|
||||
struct st_error **err)
|
||||
{
|
||||
const char *begin = lx->pos;
|
||||
const char *kw;
|
||||
size_t kwlen;
|
||||
|
||||
advance(lx); /* '#' */
|
||||
kw = lx->pos;
|
||||
if (*lx->pos == '-') {
|
||||
advance(lx);
|
||||
}
|
||||
while (at_ident_char(lx)) {
|
||||
advance(lx);
|
||||
}
|
||||
kwlen = (size_t)(lx->pos - kw);
|
||||
|
||||
if (kwlen == 3 && memcmp(kw, "inf", 3) == 0) {
|
||||
emit(out, ST_TOK_INF, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_INF;
|
||||
}
|
||||
if (kwlen == 4 && memcmp(kw, "-inf", 4) == 0) {
|
||||
emit(out, ST_TOK_NEG_INF, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_NEG_INF;
|
||||
}
|
||||
if (kwlen == 3 && memcmp(kw, "nan", 3) == 0) {
|
||||
emit(out, ST_TOK_NAN, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_NAN;
|
||||
}
|
||||
if (kwlen == 4 && memcmp(kw, "true", 4) == 0) {
|
||||
emit(out, ST_TOK_TRUE, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_TRUE;
|
||||
}
|
||||
if (kwlen == 5 && memcmp(kw, "false", 5) == 0) {
|
||||
emit(out, ST_TOK_FALSE, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_FALSE;
|
||||
}
|
||||
if (kwlen == 4 && memcmp(kw, "null", 4) == 0) {
|
||||
emit(out, ST_TOK_NULL, start, begin, (size_t)(lx->pos - begin));
|
||||
return ST_TOK_NULL;
|
||||
}
|
||||
return fail(lx, start, "unknown '#'-prefixed keyword", err);
|
||||
}
|
||||
|
||||
/* Dispatch on '#': raw string or keyword. */
|
||||
static enum st_token_kind
|
||||
lex_hash(struct st_lexer *lx, struct st_token *out, struct st_span start,
|
||||
struct st_error **err)
|
||||
{
|
||||
const char *p = lx->pos;
|
||||
|
||||
while (*p == '#') {
|
||||
p++;
|
||||
}
|
||||
if (*p == '"') {
|
||||
return lex_raw_string(lx, out, start, err);
|
||||
}
|
||||
if (lx->pos[1] == '#') {
|
||||
return fail(lx, start,
|
||||
"invalid token: '#' must be followed by a string or a keyword",
|
||||
err);
|
||||
}
|
||||
return lex_keyword(lx, out, start, err);
|
||||
}
|
||||
|
||||
/* --- type annotation --------------------------------------------------- */
|
||||
|
||||
static enum st_token_kind
|
||||
lex_type(struct st_lexer *lx, struct st_token *out, struct st_span start,
|
||||
struct st_error **err)
|
||||
{
|
||||
const char *name;
|
||||
size_t namelen;
|
||||
|
||||
advance(lx); /* '(' */
|
||||
while (is_space((unsigned char)*lx->pos)) {
|
||||
advance(lx);
|
||||
}
|
||||
if (!at_ident_start(lx)) {
|
||||
return fail(lx, start, "expected a type name inside '('", err);
|
||||
}
|
||||
name = lx->pos;
|
||||
advance(lx);
|
||||
while (at_ident_char(lx)) {
|
||||
advance(lx);
|
||||
}
|
||||
namelen = (size_t)(lx->pos - name);
|
||||
|
||||
while (is_space((unsigned char)*lx->pos)) {
|
||||
advance(lx);
|
||||
}
|
||||
if (*lx->pos != ')') {
|
||||
return fail(lx, start,
|
||||
"unterminated type annotation (expected ')')", err);
|
||||
}
|
||||
advance(lx); /* ')' */
|
||||
|
||||
emit(out, ST_TOK_TYPE, start, name, namelen);
|
||||
return ST_TOK_TYPE;
|
||||
}
|
||||
|
||||
/* --- comments ---------------------------------------------------------- */
|
||||
|
||||
static enum st_token_kind
|
||||
lex_comment(struct st_lexer *lx, struct st_token *out, struct st_span start,
|
||||
struct st_error **err)
|
||||
{
|
||||
const char *begin = lx->pos;
|
||||
|
||||
if (lx->pos[1] == '/') {
|
||||
advance(lx);
|
||||
advance(lx); /* '//' */
|
||||
while (*lx->pos != '\0' && !is_newline_at(lx->pos)) {
|
||||
advance(lx);
|
||||
}
|
||||
emit(out, ST_TOK_LINE_COMMENT, start, begin,
|
||||
(size_t)(lx->pos - begin));
|
||||
return ST_TOK_LINE_COMMENT;
|
||||
}
|
||||
if (lx->pos[1] == '*') {
|
||||
int depth = 1;
|
||||
|
||||
advance(lx);
|
||||
advance(lx); /* slash-star opener */
|
||||
while (depth > 0) {
|
||||
if (*lx->pos == '\0') {
|
||||
return fail(lx, start, "unterminated block comment", err);
|
||||
}
|
||||
if (lx->pos[0] == '/' && lx->pos[1] == '*') {
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
depth++;
|
||||
continue;
|
||||
}
|
||||
if (lx->pos[0] == '*' && lx->pos[1] == '/') {
|
||||
advance(lx);
|
||||
advance(lx);
|
||||
depth--;
|
||||
continue;
|
||||
}
|
||||
advance(lx);
|
||||
}
|
||||
emit(out, ST_TOK_BLOCK_COMMENT, start, begin,
|
||||
(size_t)(lx->pos - begin));
|
||||
return ST_TOK_BLOCK_COMMENT;
|
||||
}
|
||||
if (lx->pos[1] == '-') {
|
||||
advance(lx);
|
||||
advance(lx); /* '/-' */
|
||||
emit(out, ST_TOK_SLASHDASH, start, begin,
|
||||
(size_t)(lx->pos - begin));
|
||||
return ST_TOK_SLASHDASH;
|
||||
}
|
||||
return fail(lx, start, "unexpected '/'", err);
|
||||
}
|
||||
|
||||
/* --- public API -------------------------------------------------------- */
|
||||
|
||||
struct st_lexer *
|
||||
st_lexer_new(const char *src, const char *filename)
|
||||
{
|
||||
struct st_lexer *lx = calloc(1, sizeof(*lx));
|
||||
|
||||
if (lx == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
lx->src = src != NULL ? src : "";
|
||||
lx->pos = lx->src;
|
||||
lx->filename = filename;
|
||||
lx->line = 1;
|
||||
lx->col = 1;
|
||||
|
||||
/* Skip a leading BOM (U+FEFF = EF BB BF), legal only as the very first
|
||||
* code point of a document. */
|
||||
if ((unsigned char)lx->pos[0] == 0xEF &&
|
||||
(unsigned char)lx->pos[1] == 0xBB &&
|
||||
(unsigned char)lx->pos[2] == 0xBF) {
|
||||
lx->pos += 3;
|
||||
}
|
||||
return lx;
|
||||
}
|
||||
|
||||
void
|
||||
st_lexer_free(struct st_lexer *lx)
|
||||
{
|
||||
free(lx);
|
||||
}
|
||||
|
||||
enum st_token_kind
|
||||
st_lexer_next(struct st_lexer *lx, struct st_token *out, struct st_error **err)
|
||||
{
|
||||
unsigned char c;
|
||||
struct st_span start;
|
||||
|
||||
if (err != NULL) {
|
||||
*err = NULL;
|
||||
}
|
||||
|
||||
/* Skip whitespace, newlines, and line continuations. */
|
||||
for (;;) {
|
||||
while (*lx->pos != '\0' &&
|
||||
(is_space((unsigned char)*lx->pos) || is_newline_at(lx->pos))) {
|
||||
advance(lx);
|
||||
}
|
||||
if (*lx->pos == '\\') {
|
||||
if (lex_escline(lx, err) < 0) {
|
||||
return ST_TOK_ERROR;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (*lx->pos == '\0') {
|
||||
emit(out, ST_TOK_EOF, (struct st_span){ lx->filename, lx->line,
|
||||
lx->col }, lx->pos, 0);
|
||||
return ST_TOK_EOF;
|
||||
}
|
||||
|
||||
start = (struct st_span){ lx->filename, lx->line, lx->col };
|
||||
c = (unsigned char)*lx->pos;
|
||||
|
||||
/* numbers, signs, and dotted identifiers */
|
||||
if (is_digit(c) || c == '+' || c == '-' || c == '.') {
|
||||
return lex_value_start(lx, out, start, err);
|
||||
}
|
||||
|
||||
switch (c) {
|
||||
case '{':
|
||||
emit(out, ST_TOK_LBRACE, start, lx->pos, 1);
|
||||
advance(lx);
|
||||
break;
|
||||
case '}':
|
||||
emit(out, ST_TOK_RBRACE, start, lx->pos, 1);
|
||||
advance(lx);
|
||||
break;
|
||||
case ';':
|
||||
emit(out, ST_TOK_SEMICOLON, start, lx->pos, 1);
|
||||
advance(lx);
|
||||
break;
|
||||
case '=':
|
||||
emit(out, ST_TOK_EQUALS, start, lx->pos, 1);
|
||||
advance(lx);
|
||||
break;
|
||||
case '"':
|
||||
if (lx->pos[1] == '"' && lx->pos[2] == '"') {
|
||||
return lex_multiline_string(lx, out, start, err, false);
|
||||
}
|
||||
return lex_quoted_string(lx, out, start, err, false);
|
||||
case '#':
|
||||
return lex_hash(lx, out, start, err);
|
||||
case '(':
|
||||
return lex_type(lx, out, start, err);
|
||||
case '/':
|
||||
return lex_comment(lx, out, start, err);
|
||||
default:
|
||||
if (at_ident_start(lx)) {
|
||||
return lex_ident(lx, out, start, err);
|
||||
}
|
||||
return fail(lx, start, "unexpected character in source", err);
|
||||
}
|
||||
|
||||
return out->kind;
|
||||
}
|
||||
|
||||
const char *
|
||||
st_token_kind_name(enum st_token_kind kind)
|
||||
{
|
||||
switch (kind) {
|
||||
case ST_TOK_EOF:
|
||||
return "eof";
|
||||
case ST_TOK_ERROR:
|
||||
return "error";
|
||||
case ST_TOK_LBRACE:
|
||||
return "lbrace";
|
||||
case ST_TOK_RBRACE:
|
||||
return "rbrace";
|
||||
case ST_TOK_SEMICOLON:
|
||||
return "semicolon";
|
||||
case ST_TOK_EQUALS:
|
||||
return "equals";
|
||||
case ST_TOK_IDENT:
|
||||
return "ident";
|
||||
case ST_TOK_STRING:
|
||||
return "string";
|
||||
case ST_TOK_MULTILINE_STRING:
|
||||
return "multiline-string";
|
||||
case ST_TOK_RAW_STRING:
|
||||
return "raw-string";
|
||||
case ST_TOK_NUMBER:
|
||||
return "number";
|
||||
case ST_TOK_INF:
|
||||
return "inf";
|
||||
case ST_TOK_NEG_INF:
|
||||
return "-inf";
|
||||
case ST_TOK_NAN:
|
||||
return "nan";
|
||||
case ST_TOK_TRUE:
|
||||
return "true";
|
||||
case ST_TOK_FALSE:
|
||||
return "false";
|
||||
case ST_TOK_NULL:
|
||||
return "null";
|
||||
case ST_TOK_TYPE:
|
||||
return "type";
|
||||
case ST_TOK_LINE_COMMENT:
|
||||
return "line-comment";
|
||||
case ST_TOK_BLOCK_COMMENT:
|
||||
return "block-comment";
|
||||
case ST_TOK_SLASHDASH:
|
||||
return "slashdash";
|
||||
default:
|
||||
return "?";
|
||||
}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* lexer.h - KDL 2.0.0 tokenizer for stupidtools.
|
||||
*
|
||||
* Produces a FLAT token stream; it does NOT parse node/argument/property
|
||||
* structure (that is the parser, todo 7). Every token carries a source
|
||||
* span (struct st_span) and a borrowed text slice into the source buffer.
|
||||
*
|
||||
* API
|
||||
* ---
|
||||
* struct st_lexer *st_lexer_new(const char *src, const char *filename);
|
||||
* enum st_token_kind st_lexer_next(struct st_lexer *, struct st_token *,
|
||||
* struct st_error **err);
|
||||
* void st_lexer_free(struct st_lexer *);
|
||||
* const char *st_token_kind_name(enum st_token_kind);
|
||||
*
|
||||
* st_lexer_new() borrows `src` (never copies); the buffer must remain
|
||||
* valid for the lexer's lifetime because token text slices point into it.
|
||||
* `filename` is also borrowed and is stamped into every span.
|
||||
*
|
||||
* st_lexer_next() returns one token per call. It returns:
|
||||
* - a token kind (>= ST_TOK_LBRACE) and fills `*out` on success,
|
||||
* - ST_TOK_EOF at end of input (out->text/len are empty),
|
||||
* - ST_TOK_ERROR on a lex error, in which case `*err` is set to a
|
||||
* freshly allocated st_error (category ST_ERR_KDL_PARSE) and `*out`
|
||||
* is left unchanged. The caller owns `*err` and frees it with
|
||||
* st_error_free().
|
||||
*
|
||||
* ERROR-SPAN LIFETIME (borrowed span)
|
||||
* ---
|
||||
* st_error carries only a BORROWED span pointer (src/error.h), so the
|
||||
* lexer cannot hand back a stack-local span. Instead the lexer stores the
|
||||
* offending span in its own `err_span` field and passes &lx->err_span to
|
||||
* st_error_at(). Consequence: the returned error's span is valid only
|
||||
* until the NEXT st_lexer_next() call or st_lexer_free() — copy the span
|
||||
* (or print the error) before re-lexing. This mirrors the span.h "file is
|
||||
* borrowed" convention.
|
||||
*
|
||||
* TOKEN-ROLE NOTES (contract for todo 7, the parser)
|
||||
* ---
|
||||
* The lexer is deliberately role-agnostic. NODE_NAME / ARGUMENT / PROPERTY
|
||||
* are parser-level roles, not lexical kinds: the parser classifies
|
||||
* ST_TOK_IDENT as a node name, an argument value, or a property key
|
||||
* followed by ST_TOK_EQUALS, by position. String/raw/multi-line text is
|
||||
* the raw source slice (delimiters and escapes intact) — unescaping and
|
||||
* multi-line dedenting belong to the value model (todo 8), not here.
|
||||
*
|
||||
* Copyright (c) 2026 huntedbytheirs
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
#ifndef ST_KDL_LEXER_H
|
||||
#define ST_KDL_LEXER_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "span.h"
|
||||
|
||||
struct st_error;
|
||||
|
||||
/* Token kinds, ordered so the punctuation kinds follow ST_TOK_ERROR. */
|
||||
enum st_token_kind {
|
||||
ST_TOK_EOF = 0, /* end of input */
|
||||
ST_TOK_ERROR, /* lex error; only ever RETURNED, never stored in a token */
|
||||
|
||||
/* structural punctuation */
|
||||
ST_TOK_LBRACE, /* { children-block open */
|
||||
ST_TOK_RBRACE, /* } children-block close */
|
||||
ST_TOK_SEMICOLON, /* ; node separator */
|
||||
ST_TOK_EQUALS, /* = property assignment */
|
||||
|
||||
/* strings (KDL §3.9) */
|
||||
ST_TOK_IDENT, /* bare identifier (node name / arg / property key) */
|
||||
ST_TOK_STRING, /* "quoted" single-line string */
|
||||
ST_TOK_MULTILINE_STRING, /* """multi-line""" string */
|
||||
ST_TOK_RAW_STRING, /* #"raw"# or #"""raw multi-line"""# */
|
||||
|
||||
/* numbers (KDL §3.14) */
|
||||
ST_TOK_NUMBER, /* decimal / 0x hex / 0o octal / 0b binary */
|
||||
ST_TOK_INF, /* #inf */
|
||||
ST_TOK_NEG_INF, /* #-inf */
|
||||
ST_TOK_NAN, /* #nan */
|
||||
|
||||
/* booleans and null (KDL §3.15-3.16) */
|
||||
ST_TOK_TRUE, /* #true */
|
||||
ST_TOK_FALSE, /* #false */
|
||||
ST_TOK_NULL, /* #null */
|
||||
|
||||
/* type annotation (KDL §3.8); text slice is the inner identifier */
|
||||
ST_TOK_TYPE, /* (u8) -> text "u8" */
|
||||
|
||||
/* comments (KDL §3.17) */
|
||||
ST_TOK_LINE_COMMENT, /* // ... */
|
||||
ST_TOK_BLOCK_COMMENT, /* ... (possibly nested) */
|
||||
ST_TOK_SLASHDASH, /* /- node/entry-level comment */
|
||||
};
|
||||
|
||||
/* One token. `span` is the 1-based line/col of the token's first byte;
|
||||
* `text`/`len` are a borrowed slice into the source (NOT NUL-terminated;
|
||||
* for strings it includes the delimiters). */
|
||||
struct st_token {
|
||||
enum st_token_kind kind;
|
||||
struct st_span span;
|
||||
const char *text;
|
||||
size_t len;
|
||||
};
|
||||
|
||||
struct st_lexer;
|
||||
|
||||
struct st_lexer *st_lexer_new(const char *src, const char *filename);
|
||||
void st_lexer_free(struct st_lexer *lx);
|
||||
|
||||
enum st_token_kind st_lexer_next(struct st_lexer *lx, struct st_token *out,
|
||||
struct st_error **err);
|
||||
|
||||
/* Stable display name for a token kind (e.g. "ident"). Unknown kinds
|
||||
* yield "?" rather than indexing out of range. */
|
||||
const char *st_token_kind_name(enum st_token_kind kind);
|
||||
|
||||
#endif /* ST_KDL_LEXER_H */
|
||||
Reference in New Issue
Block a user