From 22e22f2575645c4c99153ad5d9b17660326a74e0 Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Sat, 29 Aug 2026 00:52:02 -0400 Subject: [PATCH] feat: self-host configure generation for stupidtools Wire the full pipeline into the binary: parse + validate the KDL buildfile, discover extensions, bridge Lua, detect the C toolchain, and emit ./configure. Adds the project's own stupid.kdl build file and a temp-dir-isolated self-host integration test proving the generated configure + make rebuild the tool with -std=c23. tests/run.sh tests 6+8 re-pin the interim 'accepted, no read' CLI assertions to the wired 'missing buildfile -> exit 1' behavior. --- src/Makefile.am | 10 ++- src/main.c | 208 ++++++++++++++++++++++++++++++++++++---------- stupid.kdl | 79 ++++++++++++++++++ tests/run.sh | 18 ++-- tests/selfhost.sh | 181 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 442 insertions(+), 54 deletions(-) create mode 100644 stupid.kdl create mode 100755 tests/selfhost.sh diff --git a/src/Makefile.am b/src/Makefile.am index 2715c8c..43c32ca 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,7 +6,10 @@ bin_PROGRAMS = stupidtools stupidtools_SOURCES = \ main.c cli.c \ error.c span.c \ - ext/discovery.c ext/lua.c ext/abi.c ext/lang_c.c ext/lang_cpp.c \ + ext/discovery.c ext/lua.c ext/abi.c ext/lang_c.c ext/lang_cpp.c ext/api.c \ + kdl/lexer.c kdl/parser.c kdl/value.c kdl/schema.c \ + detect/check_registry.c detect/checks.c detect/probe.c detect/resolve.c \ + gen/sh_emit.c gen/configure.c gen/config.c gen/args.c \ ../thirdparty/lua/lapi.c ../thirdparty/lua/lauxlib.c \ ../thirdparty/lua/lbaselib.c ../thirdparty/lua/lcode.c \ ../thirdparty/lua/lctype.c ../thirdparty/lua/ldebug.c \ @@ -27,4 +30,7 @@ stupidtools_SOURCES = \ # the six libm shims in lua.c keep the link free of -lm). noinst_HEADERS = cli.h error.h span.h \ ext/discovery.h ext/lua.h ext/abi.h ext/lang_c.h ext/lang_cpp.h \ - ext/lang_common.h + ext/lang_common.h ext/api.h \ + kdl/lexer.h kdl/ast.h kdl/value.h kdl/schema.h \ + detect/check_registry.h detect/checks.h detect/probe.h detect/resolve.h \ + gen/sh_emit.h gen/configure.h gen/config.h gen/args.h diff --git a/src/main.c b/src/main.c index e05df87..81a4991 100644 --- a/src/main.c +++ b/src/main.c @@ -4,25 +4,45 @@ * Copyright (c) 2026 huntedbytheirs * SPDX-License-Identifier: BSD-3-Clause * - * Entrypoint: parse argv via cli.c, dispatch --help/--version/usage - * errors, and reserve the `stupidtools ` shape. Buildfile - * processing arrives in later todos (16+). + * Entrypoint. Parses argv via cli.c, then runs the full pipeline for a + * single buildfile: * - * Interim extension pipeline (todo 20): with --ext-dir flags or a - * STUPIDTOOLS_EXT environment, run extension discovery, print what - * loaded, and exit 0 -- until todo 23 folds discovery into the full - * buildfile pipeline. + * 1. read the buildfile (argv[1]) into memory; + * 2. parse it (st_kdl_parse) and validate it against the DSL + * (st_kdl_validate); + * 3. build the extension context + sandboxed Lua runtime, discover + * every extension (builtin C modules first, then --ext-dir / + * STUPIDTOOLS_EXT / the user dir / the builtin dir -- see + * ext/discovery.h), flush Lua registrations into the ctx + * (st_ext_bridge_lua), and detect the C toolchain + * (st_ext_detect_language(ctx, "c")); + * 4. emit ./configure (st_gen_configure_emit_path) into the current + * working directory -- the generated script reads ./Makefile.in + * and writes ./Makefile there at configure time (the v1 source-dir + * contract, see gen/configure.h); + * 5. print a one-line summary and exit 0. + * + * Any failure prints its diagnostic via st_error_print to stderr and + * exits 1 (runtime error); CLI usage errors exit 2 (see cli.h). Only + * main.c exits; every library path returns an error. */ #include "cli.h" +#include "error.h" #include "ext/abi.h" +#include "ext/api.h" #include "ext/discovery.h" #include "ext/lua.h" +#include "gen/configure.h" +#include "kdl/ast.h" +#include "kdl/schema.h" +#include #include #include #include +#include /* Discovery's log sink: route the "loaded extension: ..." lines to * stdout so scripts can grep them cleanly. */ @@ -32,35 +52,78 @@ stdout_log(const char *fmt, va_list ap) vfprintf(stdout, fmt, ap); } -/* Creates a fresh ctx + runtime, discovers everything (builtin modules - * first, then the --ext-dir dirs, STUPIDTOOLS_EXT, the user dir and the - * builtin dir -- see ext/discovery.h), and prints each loaded module. - * Returns CLI_EXIT_OK on success; a load error prints its diagnostic to - * stderr and returns CLI_EXIT_RUNTIME (fail-fast contract). */ -static int -run_extension_discovery(const struct cli_opts *opts) +/* Reads the whole of `path` into a freshly malloc'd NUL-terminated + * buffer (owned by the caller). Returns NULL on success; an owned + * ST_ERR_IO error naming the file on failure (unreadable file or OOM). + * The read uses rb so embedded NULs don't truncate the parse. */ +static struct st_error * +read_file(const char *path, char **out) { - struct st_ext_ctx *ctx = st_ext_ctx_new(); - struct st_lua_rt *rt = st_lua_rt_new(); - struct st_error *err = NULL; - int rc = CLI_EXIT_RUNTIME; + FILE *f; + long size; + char *buf; + size_t got; + size_t need; + struct st_error *err; - if (ctx == NULL || rt == NULL) { - fprintf(stderr, "%s: error: out of memory\n", opts->program); - } else { - st_ext_discover_set_log(stdout_log); - err = st_ext_discover(ctx, rt, opts->ext_dirs, - opts->ext_dir_count); - if (err != NULL) { - st_error_print(stderr, err); - st_error_free(err); - } else { - rc = CLI_EXIT_OK; - } + f = fopen(path, "rb"); + if (f == NULL) { + need = strlen(path) + 64; + buf = malloc(need); + if (buf == NULL) + return st_error_internal("out of memory"); + (void)snprintf(buf, need, "cannot read build file '%s': %s", + path, strerror(errno)); + err = st_error_io(buf); + free(buf); + return err; } - st_lua_rt_free(rt); - st_ext_ctx_free(ctx); - return rc; + if (fseek(f, 0, SEEK_END) != 0) { + (void)fclose(f); + need = strlen(path) + 64; + buf = malloc(need); + if (buf == NULL) + return st_error_internal("out of memory"); + (void)snprintf(buf, need, "cannot read build file '%s': %s", + path, strerror(errno)); + err = st_error_io(buf); + free(buf); + return err; + } + size = ftell(f); + if (size < 0 || fseek(f, 0, SEEK_SET) != 0) { + (void)fclose(f); + need = strlen(path) + 64; + buf = malloc(need); + if (buf == NULL) + return st_error_internal("out of memory"); + (void)snprintf(buf, need, "cannot read build file '%s': %s", + path, strerror(errno)); + err = st_error_io(buf); + free(buf); + return err; + } + buf = malloc((size_t)size + 1); + if (buf == NULL) { + (void)fclose(f); + return st_error_internal("out of memory"); + } + got = fread(buf, 1, (size_t)size, f); + if (got != (size_t)size || fclose(f) != 0) { + free(buf); + need = strlen(path) + 64; + buf = malloc(need); + if (buf == NULL) + return st_error_internal("out of memory"); + (void)snprintf(buf, need, "cannot read build file '%s': %s", + path, strerror(errno)); + err = st_error_io(buf); + free(buf); + return err; + } + buf[size] = '\0'; + *out = buf; + return NULL; } int @@ -68,7 +131,12 @@ main(int argc, char **argv) { struct cli_opts opts; enum cli_action action; - const char *env; + struct st_error *err = NULL; + char *src = NULL; + struct st_kdl_document *doc = NULL; + struct st_ext_ctx *ctx = NULL; + struct st_lua_rt *rt = NULL; + int rc = CLI_EXIT_RUNTIME; action = cli_parse(argc, argv, &opts); @@ -84,14 +152,6 @@ main(int argc, char **argv) break; } - if (opts.ext_dir_count > 0) { - return run_extension_discovery(&opts); - } - env = getenv("STUPIDTOOLS_EXT"); - if (env != NULL && env[0] != '\0') { - return run_extension_discovery(&opts); - } - if (opts.buildfile == NULL) { fprintf(stderr, "%s: error: no build file given\n", opts.program); @@ -99,7 +159,65 @@ main(int argc, char **argv) return CLI_EXIT_RUNTIME; } - /* Accepted: the buildfile is parsed in todo 16+, not read here. */ - (void)opts.buildfile; - return CLI_EXIT_OK; + /* 1. read the buildfile. */ + err = read_file(opts.buildfile, &src); + if (err != NULL) + goto fail; + + /* 2. parse + validate. `src` must outlive the document (the AST + * borrows its token slices), so it is freed last. */ + doc = st_kdl_parse(src, opts.buildfile, &err); + if (doc == NULL) + goto fail; + + err = st_kdl_validate(doc); + if (err != NULL) + goto fail; + + /* 3. extensions + toolchain. Discovery runs the builtin modules + * itself (st_ext_init_builtins), so we never call it here. */ + ctx = st_ext_ctx_new(); + rt = st_lua_rt_new(); + if (ctx == NULL || rt == NULL) { + err = st_error_internal("out of memory"); + goto fail; + } + + st_ext_discover_set_log(stdout_log); + err = st_ext_discover(ctx, rt, opts.ext_dirs, opts.ext_dir_count); + if (err != NULL) + goto fail; + + err = st_ext_bridge_lua(ctx, rt); + if (err != NULL) + goto fail; + + err = st_ext_detect_language(ctx, "c"); + if (err != NULL) + goto fail; + + /* 4. emit ./configure into the current working directory. */ + err = st_gen_configure_emit_path("configure", doc, ctx); + if (err != NULL) + goto fail; + + /* 5. summary. */ + { + const char *cc = st_registry_get_var(st_ext_var_registry(ctx), + "CC"); + printf("wrote ./configure (CC=%s)\n", + cc != NULL ? cc : "cc"); + } + rc = CLI_EXIT_OK; + +fail: + if (err != NULL) { + st_error_print(stderr, err); + st_error_free(err); + } + st_lua_rt_free(rt); + st_ext_ctx_free(ctx); + st_kdl_document_free(doc); + free(src); + return rc; } diff --git a/stupid.kdl b/stupid.kdl new file mode 100644 index 0000000..0c1c334 --- /dev/null +++ b/stupid.kdl @@ -0,0 +1,79 @@ +/* stupid.kdl - stupidtools' OWN build file (todo 23: self-host). + * + * The project describes itself in its own DSL. This is a sibling of + * tests/fixtures/stupid.kdl (the todo-9 grammar fixture): richer, with the + * REAL source list, so `src/stupidtools stupid.kdl` generates a ./configure + * that can rebuild the tool end to end. The autotools path (configure.ac + + * Makefile.am) remains the bootstrap; this file is what the self-host test + * (tests/selfhost.sh) drives. + * + * DSL grammar (src/kdl/schema.h): `project` first, then targets/features/ + * options. The `target` source list is descriptive in v1 (the generated + * configure substitutes @VAR@ into a user-provided Makefile.in; it does not + * generate a Makefile) but lists every translation unit so the self-host + * Makefile.in can compile them all. */ + +project "stupidtools" version "1.0.0" + +// The POSIX surface the tool itself relies on (fork/exec/pipe/readdir for +// detection and discovery). A real check: the generated configure probes +// unistd.h with the detected C compiler. +feature "posix" { + header "unistd.h" +} + +// The one artifact: every translation unit of the tool, plus the posix +// feature it needs at build time. +target "default" { + src "src/main.c" + src "src/cli.c" + src "src/error.c" + src "src/span.c" + src "src/kdl/lexer.c" + src "src/kdl/parser.c" + src "src/kdl/value.c" + src "src/kdl/schema.c" + src "src/detect/check_registry.c" + src "src/detect/checks.c" + src "src/detect/probe.c" + src "src/detect/resolve.c" + src "src/gen/sh_emit.c" + src "src/gen/configure.c" + src "src/gen/config.c" + src "src/gen/args.c" + src "src/ext/discovery.c" + src "src/ext/lua.c" + src "src/ext/abi.c" + src "src/ext/api.c" + src "src/ext/lang_c.c" + src "src/ext/lang_cpp.c" + src "thirdparty/lua/lapi.c" + src "thirdparty/lua/lauxlib.c" + src "thirdparty/lua/lbaselib.c" + src "thirdparty/lua/lcode.c" + src "thirdparty/lua/lctype.c" + src "thirdparty/lua/ldebug.c" + src "thirdparty/lua/ldo.c" + src "thirdparty/lua/ldump.c" + src "thirdparty/lua/lfunc.c" + src "thirdparty/lua/lgc.c" + src "thirdparty/lua/llex.c" + src "thirdparty/lua/lmem.c" + src "thirdparty/lua/lobject.c" + src "thirdparty/lua/lopcodes.c" + src "thirdparty/lua/lparser.c" + src "thirdparty/lua/lstate.c" + src "thirdparty/lua/lstring.c" + src "thirdparty/lua/ltable.c" + src "thirdparty/lua/ltm.c" + src "thirdparty/lua/lundump.c" + src "thirdparty/lua/lvm.c" + src "thirdparty/lua/lzio.c" + src "thirdparty/lua/lstrlib.c" + src "thirdparty/lua/ltablib.c" + feature "posix" +} + +// --enable-debug / --disable-debug (default off), like configure.ac's +// release/debug split. +option "debug" default=#false diff --git a/tests/run.sh b/tests/run.sh index f31a95a..1f3e285 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -98,11 +98,13 @@ else fail "no arguments exited rc=$noargs_rc (want 1): $noargs_out" fi -# --- 6. one positional buildfile must be accepted (exit 0, no read) ------ -if "$BIN" buildfile.kdl >/dev/null 2>&1; then - pass "one positional buildfile is accepted (exit 0)" +# --- 6. a missing buildfile is a runtime error (exit 1) ------------------- +"$BIN" buildfile.kdl >/dev/null 2>&1 +rc=$? +if [ "$rc" -eq 1 ]; then + pass "missing buildfile exits 1 (runtime error)" else - fail "one positional buildfile was rejected (rc=$?)" + fail "missing buildfile exited rc=$rc (want 1)" fi # --- 7. --version=1 is an unknown option form -> exit 2 ------------------- @@ -115,10 +117,12 @@ else fi # --- 8. -- ends option parsing; following text is positional -------------- -if "$BIN" -- --bogus >/dev/null 2>&1; then - pass "-- terminates option parsing (--bogus taken as buildfile)" +"$BIN" -- --bogus >/dev/null 2>&1 +rc=$? +if [ "$rc" -eq 1 ]; then + pass "-- terminates option parsing (--bogus taken as a missing buildfile: rc=1, not 2)" else - fail "-- --bogus exited non-zero (rc=$?)" + fail "-- --bogus exited rc=$rc (want 1, proving --bogus was a buildfile not an option)" fi # --- 9. two positionals are a usage error -> exit 2 ----------------------- diff --git a/tests/selfhost.sh b/tests/selfhost.sh new file mode 100755 index 0000000..ec381a2 --- /dev/null +++ b/tests/selfhost.sh @@ -0,0 +1,181 @@ +#!/bin/sh +# tests/selfhost.sh - self-host integration test (plan todo 23). +# +# Proves stupidtools generates its OWN ./configure end to end: the tool +# (already built by `make`) reads the repo's stupid.kdl, emits ./configure +# into a TEMP dir, that configure produces a Makefile, and `make` builds a +# working stupidtools whose compile line used -std=c23. The generated +# configure/Makefile/binary live in the temp dir ONLY - the repo's own +# autotools bootstrap (configure.ac + Makefile.am) is never touched, and no +# in-tree residue is produced. +# +# POSIX sh only (dash/bash/zsh safe); no [[ ]], arrays, local, ==, <<<, &>. +# Exit 0 = pass, non-zero = fail. Fails fast on a hard setup error. + +TOP_DIR=$(cd "$(dirname "$0")/.." && pwd) || { + echo "FAIL: cannot resolve project root" >&2 + exit 1 +} +BIN="$TOP_DIR/src/stupidtools" + +TESTS_RUN=0 +TESTS_FAILED=0 + +pass() { + TESTS_RUN=$((TESTS_RUN + 1)) + printf 'ok %d - %s\n' "$TESTS_RUN" "$1" +} + +fail() { + TESTS_RUN=$((TESTS_RUN + 1)) + TESTS_FAILED=$((TESTS_FAILED + 1)) + printf 'not ok %d - %s\n' "$TESTS_RUN" "$1" >&2 +} + +# --- 1. the built binary must exist -------------------------------------- +if [ -x "$BIN" ]; then + pass "binary exists and is executable: $BIN" +else + fail "binary missing: $BIN (run 'make' first)" + printf 'FAILED: %d/%d checks failed\n' "$TESTS_FAILED" "$TESTS_RUN" >&2 + exit 1 +fi + +# --- temp dir (isolated; cleaned on every exit path) --------------------- +WORK=$(mktemp -d "${TMPDIR:-/tmp}/stupidtools-selfhost.XXXXXX") || { + fail "cannot create temp dir" + exit 1 +} +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT HUP INT TERM + +# --- 2. copy stupid.kdl + write a Makefile.in into the temp dir ---------- +cp "$TOP_DIR/stupid.kdl" "$WORK/stupid.kdl" || { + fail "cannot copy stupid.kdl into temp dir" + exit 1 +} + +# The tool's own source list (repo-relative, kept in sync with stupid.kdl's +# `target` and src/Makefile.am). Prefixed with $TOP_DIR so the self-hosted +# build compiles the REAL sources from the isolated temp dir. The `\$(...)` +# escapes keep CC/CFLAGS/LIBS as MAKE variables (substituted by configure); +# `$SRCS` expands here (shell) into the absolute path list. +SRCS="$TOP_DIR/src/main.c $TOP_DIR/src/cli.c $TOP_DIR/src/error.c $TOP_DIR/src/span.c \ +$TOP_DIR/src/kdl/lexer.c $TOP_DIR/src/kdl/parser.c $TOP_DIR/src/kdl/value.c $TOP_DIR/src/kdl/schema.c \ +$TOP_DIR/src/detect/check_registry.c $TOP_DIR/src/detect/checks.c $TOP_DIR/src/detect/probe.c $TOP_DIR/src/detect/resolve.c \ +$TOP_DIR/src/gen/sh_emit.c $TOP_DIR/src/gen/configure.c $TOP_DIR/src/gen/config.c $TOP_DIR/src/gen/args.c \ +$TOP_DIR/src/ext/discovery.c $TOP_DIR/src/ext/lua.c $TOP_DIR/src/ext/abi.c $TOP_DIR/src/ext/api.c \ +$TOP_DIR/src/ext/lang_c.c $TOP_DIR/src/ext/lang_cpp.c \ +$TOP_DIR/thirdparty/lua/lapi.c $TOP_DIR/thirdparty/lua/lauxlib.c \ +$TOP_DIR/thirdparty/lua/lbaselib.c $TOP_DIR/thirdparty/lua/lcode.c \ +$TOP_DIR/thirdparty/lua/lctype.c $TOP_DIR/thirdparty/lua/ldebug.c \ +$TOP_DIR/thirdparty/lua/ldo.c $TOP_DIR/thirdparty/lua/ldump.c \ +$TOP_DIR/thirdparty/lua/lfunc.c $TOP_DIR/thirdparty/lua/lgc.c \ +$TOP_DIR/thirdparty/lua/llex.c $TOP_DIR/thirdparty/lua/lmem.c \ +$TOP_DIR/thirdparty/lua/lobject.c $TOP_DIR/thirdparty/lua/lopcodes.c \ +$TOP_DIR/thirdparty/lua/lparser.c $TOP_DIR/thirdparty/lua/lstate.c \ +$TOP_DIR/thirdparty/lua/lstring.c $TOP_DIR/thirdparty/lua/ltable.c \ +$TOP_DIR/thirdparty/lua/ltm.c $TOP_DIR/thirdparty/lua/lundump.c \ +$TOP_DIR/thirdparty/lua/lvm.c $TOP_DIR/thirdparty/lua/lzio.c \ +$TOP_DIR/thirdparty/lua/lstrlib.c $TOP_DIR/thirdparty/lua/ltablib.c" + +cat > "$WORK/Makefile.in" <gen.log 2>&1); then + if [ -s "$WORK/configure" ]; then + pass "stupidtools generated a non-empty ./configure" + else + fail "stupidtools exited 0 but wrote no ./configure (see $WORK/gen.log)" + fi +else + rc=$? + fail "stupidtools failed on stupid.kdl (rc=$rc): $(cat "$WORK/gen.log" 2>/dev/null)" +fi + +# --- 4. run ./configure -> Makefile -------------------------------------- +if [ -s "$WORK/configure" ]; then + if sh -n "$WORK/configure" 2>"$WORK/configure.n.err"; then + pass "generated ./configure passes sh -n (POSIX syntax)" + else + fail "generated ./configure fails sh -n: $(cat "$WORK/configure.n.err" 2>/dev/null)" + fi + + if (cd "$WORK" && sh ./configure >config.log 2>&1); then + if [ -s "$WORK/Makefile" ]; then + pass "./configure ran and wrote a non-empty Makefile" + else + fail "./configure exited 0 but wrote no Makefile (see $WORK/config.log)" + fi + else + rc=$? + fail "./configure failed (rc=$rc): $(cat "$WORK/config.log" 2>/dev/null)" + fi +else + fail "skipping configure run (no ./configure was generated)" +fi + +# --- 5+6. build with make V=1 and assert -std=c23 ------------------------ +if [ -s "$WORK/Makefile" ]; then + if command -v timeout >/dev/null 2>&1; then + timeout 300 make -C "$WORK" V=1 >"$WORK/make.log" 2>&1 + make_rc=$? + else + make -C "$WORK" V=1 >"$WORK/make.log" 2>&1 + make_rc=$? + fi + if [ "$make_rc" -eq 0 ] && [ -x "$WORK/stupidtools" ]; then + pass "make built a stupidtools binary in the temp dir" + else + fail "make failed (rc=$make_rc): $(tail -n 5 "$WORK/make.log" 2>/dev/null)" + fi + + if grep -q -- '-std=c23' "$WORK/make.log"; then + pass "self-hosted compile line used -std=c23" + else + fail "compile line did NOT contain -std=c23 (see $WORK/make.log)" + fi +else + fail "skipping make (no Makefile was written)" +fi + +# --- 7. the self-hosted binary must run ---------------------------------- +if [ -x "$WORK/stupidtools" ]; then + if version_out=$("$WORK/stupidtools" --version 2>&1); then + if [ "$version_out" = "stupidtools 1.0.0" ]; then + pass "self-hosted binary --version reports: $version_out" + else + fail "self-hosted --version unexpected: '$version_out'" + fi + else + rc=$? + fail "self-hosted --version exited non-zero (rc=$rc): $version_out" + fi +else + fail "skipping --version check (no self-hosted binary was built)" +fi + +# --- summary ------------------------------------------------------------- +if [ "$TESTS_FAILED" -ne 0 ]; then + printf 'FAILED: %d/%d checks failed\n' "$TESTS_FAILED" "$TESTS_RUN" >&2 + exit 1 +fi +printf 'All %d self-host checks passed.\n' "$TESTS_RUN"