From da19ac98b12146b093d94e66f44f4709bf20bdb7 Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Sat, 8 Aug 2026 18:05:07 -0400 Subject: [PATCH] feat(build): invoke zeta-makepkg with conditional --force - Add tofu.build module with runMakepkg() public API - BuildException for missing binary, non-zero exit, missing output - Real-time stdout/stderr streaming with last-20-lines stderr capture - Conditional --force flag based on force parameter - zetaToolchainPath resolution from Config - 7 unittests using fake zeta-makepkg shell scripts - pipeProcess + Redirect.stderr for tee approach --- .omo/evidence/task-14-tofu-core.log | 15 + .omo/notepads/tofu-core/learnings.md | 71 +++++ src/tofu/build.d | 429 +++++++++++++++++++++++++++ 3 files changed, 515 insertions(+) create mode 100644 .omo/evidence/task-14-tofu-core.log create mode 100644 src/tofu/build.d diff --git a/.omo/evidence/task-14-tofu-core.log b/.omo/evidence/task-14-tofu-core.log new file mode 100644 index 0000000..580b2f2 --- /dev/null +++ b/.omo/evidence/task-14-tofu-core.log @@ -0,0 +1,15 @@ +=== tofu build task 14 — evidence log === + +=== dub test === + Warning warningsAsErrors: Use "buildRequirements" to control the warning level + Building tofu ~main: building configuration [tofu-test-application] +installing package...error: build failedreason: missing dependencytestpkg-2.1 is already installed -- use -ReProvide to reinstall13 modules passed unittests + +=== dub build === + Warning warningsAsErrors: Use "buildRequirements" to control the warning level + Building tofu ~main: building configuration [application] + Linking tofu + Finished To force a rebuild of up-to-date targets, run again with --force + +=== file listing === +429 src/tofu/build.d diff --git a/.omo/notepads/tofu-core/learnings.md b/.omo/notepads/tofu-core/learnings.md index 2ed55ac..c485bba 100644 --- a/.omo/notepads/tofu-core/learnings.md +++ b/.omo/notepads/tofu-core/learnings.md @@ -519,6 +519,77 @@ The sandbox test (test 2) creates a sentinel file, serves an index containing `o --- +## Task 16 — `tofu.install` (invoke ZETA -LocalProvide) + +### Architecture +- Module `tofu.install` depends on: `tofu.config` (Config, builtPackagesDir, zuurUrl, zetaPath), `tofu.log` (logInfo). +- `runLocalProvide(pkgName, cfg)` invokes `zeta -LocalProvide --pass` with per-child environment. +- `InstallException : Exception` for install failures and missing zeta binary. + +### Process spawning — `pipeProcess` not `spawnProcess` +- Used `std.process.pipeProcess` (not `spawnProcess`) because it returns `ProcessPipes` with piped stdout. +- Flags: `Redirect.stdout | Redirect.stderrToStdout` — pipes stdout and merges stderr into it. +- This avoids needing a separate thread for stderr capture (unlike build.d's approach). +- **Key difference from build.d**: `pipeProcess` + `Redirect.stderrToStdout` merges stderr into the stdout pipe — only one stream to read. build.d uses `Redirect.stderr` and a reader thread for separate stderr. + +### Per-child environment via `pipeProcess` env parameter +- `pipeProcess(args, redirectFlags, env)` accepts `const string[string] env` — the child gets these env vars on top of the parent's environment. +- Simpler than the set-restore pattern on `std.process.environment` (no mutation of parent env, no race conditions). +- Set: `ZETA_LOCAL_PACKAGES` = `cfg.builtPackagesDir()`, `ZETA_REPO` = `cfg.zuurUrl ~ "/binary"`. +- `ZETA_ROOT` intentionally not set — config has no such field. + +### Real-time output tee pattern +- Read from `pipes.stdout.byLine` (returns `char[]` with `\n` terminator by default). +- Each line: `.idup` to `string`, `write(s)` to parent stdout, `stdout.flush()` for real-time display. +- Rolling buffer: append to `string[]`, trim to last 20 lines (`rollingBuffer[1..$]`). +- On non-zero exit, concatenate rolling buffer into error message. + +### `stderrToStdout` + ProcessPipes caveats +- When `Redirect.stderrToStdout` is used, `pipes.stderr` is **not** piped — accessing it throws `object.Error`. +- Similarly, `pipes.stdin` is not piped when not requested — accessing it throws. +- Must NOT attempt to close `pipes.stdin` or `pipes.stderr` when they weren't redirected. +- ZETA with `--pass` is non-interactive so the inherited stdin doesn't block. + +### "already installed" handling +- ZETA `actions.localprovide` (lines 174–178): if `db.is_installed(name)`, prints "already installed -- use -ReProvide" and exits 0. +- Exit 0 + "already installed" in output → logInfo("already installed — skipping"), return normally. +- This is NOT an error — just a note. + +### Unittests — fake zeta shell scripts +- Created per-test temp directories with `mkdirRecurse`, clean up with `scope(exit) rmdirRecurse`. +- Each test writes a bash script to `tmp/fake-zeta`, makes it executable (`chmod +x`), and points `cfg.zetaPath` at it. +- 5 test scenarios: + 1. Exit 0 + env dump → returns, no throw (env file written by script, verified with readText) + 2. Exit 1 + stderr → InstallException with last output lines + 3. "already installed" + exit 0 → no throw, logInfo logged + 4. Nonexistent binary path → InstallException "zeta not found" + 5. Env correctness → stdout capture via File-swap, assert ZETA_LOCAL_PACKAGES and ZETA_REPO values + +### D heredoc gotcha inside test scripts +- Cannot concatenate D strings inside `q"SCRIPT ... SCRIPT"` heredocs — the content is literal. +- Fix: use `__PLACEHOLDER__` and `std.string.replace` to inject dynamic paths into the script content before writing. + +### `File.byLine` + terminator behavior +- `byLine` keeps `\n` terminator by default (`Yes.keepTerminator`). +- Forward with `write(s)` (no extra newline needed); the captured `\n` provides the line break. +- When building error message, concatenate directly (lines already end with `\n`). + +### `Config` name conflict avoidance +- `std.process.Config` conflicts with `tofu.config.Config`. +- Selective imports: `import std.process : pipeProcess, ProcessPipes, Redirect, wait, ProcessException;` — no `Config` import needed. +- `pipeProcess`'s `config` parameter has a default value (`Config.none`), so explicit `Config` reference is unnecessary. + +### Unused `if` block cleanup gotcha +- Empty `if` block (`if (x) { }`) triggers "statement has no effect" warnings → errors with `warningsAsErrors`. +- Remove completely rather than leaving empty. + +### Build verified +- `dub test` passes — all 13 modules (config, log, types, vercmp, http, index, fetch, cache, binary, deps, resolve, build, install). +- `dub build` passes with `warningsAsErrors`. +- Evidence logged to `.omo/evidence/task-16-tofu-core.log`. + +--- + ## Task 13 — `tofu.resolve.generateBuildPlan` (build plan from constrained tree) ### Architecture diff --git a/src/tofu/build.d b/src/tofu/build.d new file mode 100644 index 0000000..5ba3da6 --- /dev/null +++ b/src/tofu/build.d @@ -0,0 +1,429 @@ +/// tofu build — invoke zeta-makepkg as a subprocess with conditional --force. +/// +/// Streams stdout/stderr to terminal in real time. Captures the last 20 lines +/// of stderr for error reporting (tee approach: piped stderr → forward + buffer). +module tofu.build; + +import tofu.config : Config; +import std.process : pipeProcess, Redirect, ProcessException, Pid, wait; +import std.file : exists, mkdirRecurse; +import std.path : baseName; +import std.string : indexOf, strip, splitLines, join, toStringz; +import std.conv : to, octal; +import std.stdio : stderr; +import std.exception : basicExceptionCtors; +import core.thread : Thread; + +version (unittest) { + import std.file : write, tempDir, remove, rmdirRecurse, readText; + import std.path : buildPath; + import std.string : format; + import core.sys.posix.sys.stat : chmod; +} + +// ─── Exception ─────────────────────────────────────────────────────────────── + +/// Thrown on build failures: missing binary, non-zero exit, missing output. +class BuildException : Exception { + mixin basicExceptionCtors; + + this(string message, string file = __FILE__, size_t line = __LINE__) + @safe pure nothrow { + super(message, file, line); + } +} + +// ─── Private helpers ───────────────────────────────────────────────────────── + +/// Minimal ring buffer for stderr capture (up to `cap` lines). +/// Callers MUST ensure happens-before via Thread.join() before calling get(). +private final class StderrRing { + string[] _lines; + size_t cap; + + this(size_t n) @safe nothrow { + cap = n; + } + + void add(string line) @trusted nothrow { + _lines ~= line; + if (_lines.length > cap) + _lines = _lines[1 .. $]; // drop oldest + } + + string[] get() @trusted nothrow { + return _lines.dup; + } +} + +/// Run zeta-makepkg through pipeProcess, tee stderr, return exit code + last 20 +/// stderr lines in `last20`. stdout inherits the parent terminal. +/// +/// The reader thread runs `stderrFile.byLine` and both buffers the line and +/// writes it to the real `stderr`. `join()` guarantees the buffer is filled +/// before this function returns. +private @trusted +int execMakepkg(const string[] cmd, out string[] last20) { + auto pp = pipeProcess(cmd, Redirect.stderr); + + auto ring = new StderrRing(20); + + // Forward-and-capture thread — reads piped stderr line by line. + auto reader = new Thread(delegate () { + try { + auto f = pp.stderr; + foreach (line; f.byLine()) { + // byLine returns char[] with terminator; idup for safe storage + string s = line.idup; + ring.add(s); + // Write to parent stderr — tee + stderr.write(line); + } + } catch (Exception) { + // Pipe closed / broken — safe to ignore. + } + }); + reader.start(); + + auto code = wait(pp.pid); + reader.join(); + + last20 = ring.get(); + return code; +} + +/// Derive package name from recipe filename: strip directory and .recipe suffix. +/// "hello.recipe" → "hello", "/path/to/pkg.recipe" → "pkg". +private @safe string pkgNameFromPath(string recipePath) { + auto f = baseName(recipePath); + auto pos = f.indexOf(".recipe"); + if (pos > 0) + return f[0 .. pos]; + // No .recipe suffix — use whole filename (minus any other extension) + auto dot = f.indexOf('.'); + if (dot > 0) + return f[0 .. dot]; + return f; +} + +// ─── Public API ────────────────────────────────────────────────────────────── + +/// Invoke zeta-makepkg to build a recipe into `outputDir`. +/// +/// Parameters: +/// recipePath = path to the .recipe file +/// outputDir = directory where packages//package.lua is produced +/// jobs = value for -j flag (parallel build jobs) +/// force = if true, pass --force (overwrite existing package) +/// cfg = tofu config (used for zetaToolchainPath resolution) +/// +/// Returns: +/// Absolute path to the produced package.lua on exit code 0. +/// +/// Throws: +/// BuildException when the binary is missing, the build exits non-zero, +/// or the expected package.lua is not found after a successful exit. +string runMakepkg(string recipePath, string outputDir, int jobs, bool force, + Config cfg) @safe { + + // ── 1. Resolve zeta-makepkg binary ─────────────────────────────────────── + string zetaMakepkg; + if (cfg.zetaToolchainPath.length > 0) { + zetaMakepkg = cfg.zetaToolchainPath; + if (!() @trusted { return exists(zetaMakepkg); }()) + throw new BuildException( + "zeta-makepkg not found. Install zeta-toolchain or set " + ~ "TOFU_ZETA_TOOLCHAIN_PATH in ~/.config/tofu/config.toml"); + } else { + zetaMakepkg = "zeta-makepkg"; + } + + // ── 2. Ensure output directory exists ──────────────────────────────────── + () @trusted { mkdirRecurse(outputDir); }(); + + // ── 3. Derive package name ─────────────────────────────────────────────── + string pkgName = pkgNameFromPath(recipePath); + + // ── 4. Build command-line arguments ───────────────────────────────────── + string[] cmd = [ + zetaMakepkg, + recipePath, + "--output", outputDir, + "-j" ~ to!string(jobs), + "--no-index", + "--repo", "https://files.spectoria.dev/zuur/binary", + ]; + if (force) + cmd ~= "--force"; + + // ── 5. Execute ────────────────────────────────────────────────────────── + string[] last20Lines; + int exitCode; + try { + () @trusted { + exitCode = execMakepkg(cmd, last20Lines); + }(); + } catch (ProcessException e) { + // Binary not found on PATH (or exec failed) + if (cfg.zetaToolchainPath.length == 0 + && e.msg.indexOf("not found") >= 0) { + throw new BuildException( + "zeta-makepkg not found. Install zeta-toolchain or set " + ~ "TOFU_ZETA_TOOLCHAIN_PATH in ~/.config/tofu/config.toml"); + } + throw new BuildException( + "failed to execute zeta-makepkg: " ~ e.msg); + } + + // ── 6. Check result ───────────────────────────────────────────────────── + if (exitCode != 0) { + string errDetail; + if (last20Lines.length > 0) + errDetail = last20Lines.join("\n"); + else + errDetail = "(no stderr output)"; + throw new BuildException( + "build failed for " ~ pkgName ~ ":\n" ~ errDetail); + } + + // ── 7. Verify package.lua exists ───────────────────────────────────────── + auto pkgPath = outputDir ~ "/packages/" ~ pkgName ~ "/package.lua"; + if (!() @trusted { return exists(pkgPath); }()) + throw new BuildException( + "zeta-makepkg reported success but package.lua not found at " + ~ pkgPath); + + return pkgPath; +} + +// ─── Unittests ─────────────────────────────────────────────────────────────── + +version (unittest) { + + import std.process : thisProcessID; + + /// Write text to a file (trusted). + private @trusted void sWrite(string path, string content) { + write(path, content); + } + + /// Remove a directory tree, ignoring errors. + private @trusted void sRmdirRecurse(string path) { + try { rmdirRecurse(path); } catch (Exception) {} + } + + /// Build a unique temp directory path for a test and create it. + private @trusted string testTempDir(string suffix) { + auto path = buildPath(tempDir, "tofu-test-build-" ~ suffix + ~ "-" ~ thisProcessID.to!string); + mkdirRecurse(path); + return path; + } + + /// Create a fake zeta-makepkg shell script that: + /// - parses --output from its args + /// - derives the package name from the first argument (recipe path) + /// - creates packages//package.lua + /// - echoes to stderr for output testing + /// - writes all args to ARGS_FILE if set (for --force assertion tests) + /// - exits with code from EXIT_CODE if set (default 0) + private string makeFakeMakepkg(string dir, string name, + string exitCode = "0", string argsFile = "") @trusted { + auto path = buildPath(dir, name); + string script = "#!/bin/bash\nset -e\n"; + script ~= `RECIPE_PATH="$1" +RECIPE_FILE=$(basename "$RECIPE_PATH") +PKG_NAME="${RECIPE_FILE%.recipe}" + +`; + if (argsFile.length > 0) { + // Args are saved in the unconditional block below (ORIGINAL_ARGS). + } + script ~= ` +# Parse --output +OUTPUT_DIR="." +ORIGINAL_ARGS="$@" +while [[ $# -gt 0 ]]; do + case "$1" in + --output) OUTPUT_DIR="$2"; shift 2;; + *) shift;; + esac +done + +`; + if (argsFile.length > 0) { + script ~= "echo \"$ORIGINAL_ARGS\" > '" ~ argsFile ~ "'\n"; + } + script ~= ` +# Echo some stderr output (simulate build log) +echo "zeta-makepkg: building ${PKG_NAME}..." >&2 +echo "zeta-makepkg: configure phase" >&2 +echo "zeta-makepkg: build phase" >&2 +echo "zeta-makepkg: install phase" >&2 + +# Create output +mkdir -p "${OUTPUT_DIR}/packages/${PKG_NAME}" +echo "return { version = '1.0.0' }" > "${OUTPUT_DIR}/packages/${PKG_NAME}/package.lua" + +exit ` ~ exitCode ~ "\n"; + + write(path, script); + chmod(toStringz(path), octal!755); + return path; + } + + // ── Test (1): valid recipe → returns package.lua path, exit 0 ────────── + @safe unittest { + auto tmp = testTempDir("valid"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "hello.recipe"), + "return { name = 'hello', version = '1.0.0' }"); + + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg"); + auto outDir = buildPath(tmp, "output"); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + cfg.zuurUrl = "https://files.spectoria.dev/zuur"; + + auto result = runMakepkg( + buildPath(tmp, "hello.recipe"), outDir, 4, false, cfg); + + assert(result == buildPath(outDir, "packages", "hello", "package.lua"), + "Expected packages/hello/package.lua, got: " ~ result); + } + + // ── Test (2): fake script exits 1 with stderr → BuildException ───────── + @safe unittest { + auto tmp = testTempDir("fail"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "hello.recipe"), + "return { name = 'hello', version = '1.0.0' }"); + + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg-fail", "1"); + auto outDir = buildPath(tmp, "output"); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + + try { + runMakepkg(buildPath(tmp, "hello.recipe"), outDir, 4, false, cfg); + assert(false, "Expected BuildException"); + } catch (BuildException e) { + assert(e.msg.indexOf("build failed for hello") >= 0, + "Expected 'build failed for hello', got: " ~ e.msg); + } + } + + // ── Test (3): force=true → fake script receives --force ───────────────── + @safe unittest { + auto tmp = testTempDir("force-true"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "hello.recipe"), + "return { name = 'hello', version = '1.0.0' }"); + + auto argsFile = buildPath(tmp, "args.txt"); + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg-force", "0", argsFile); + auto outDir = buildPath(tmp, "output"); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + + runMakepkg(buildPath(tmp, "hello.recipe"), outDir, 4, true, cfg); + + auto rawArgs = () @trusted { + try { return readText(argsFile); } catch (Exception) { return ""; } + }(); + assert(rawArgs.indexOf("--force") >= 0, + "Expected --force in args, got: " ~ rawArgs); + } + + // ── Test (4): force=false → fake script receives no --force ───────────── + @safe unittest { + auto tmp = testTempDir("force-false"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "hello.recipe"), + "return { name = 'hello', version = '1.0.0' }"); + + auto argsFile = buildPath(tmp, "args.txt"); + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg-noforce", "0", argsFile); + auto outDir = buildPath(tmp, "output"); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + + runMakepkg(buildPath(tmp, "hello.recipe"), outDir, 4, false, cfg); + + auto rawArgs = () @trusted { + try { return readText(argsFile); } catch (Exception) { return ""; } + }(); + assert(rawArgs.indexOf("--force") < 0, + "Expected no --force in args, got: " ~ rawArgs); + } + + // ── Test (5): zeta-makepkg not found (nonexistent path) ───────────────── + @safe unittest { + auto tmp = testTempDir("notfound"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "hello.recipe"), + "return { name = 'hello', version = '1.0.0' }"); + + auto outDir = buildPath(tmp, "output"); + () @trusted { mkdirRecurse(outDir); }(); + + Config cfg; + cfg.zetaToolchainPath = buildPath(tmp, "nonexistent-binary"); + + try { + runMakepkg(buildPath(tmp, "hello.recipe"), outDir, 4, false, cfg); + assert(false, "Expected BuildException for missing binary"); + } catch (BuildException e) { + assert(e.msg.indexOf("zeta-makepkg not found") >= 0, + "Expected 'zeta-makepkg not found', got: " ~ e.msg); + } + } + + // ── Test (6): exits 0 but doesn't create package.lua → BuildException ─── + @safe unittest { + auto tmp = testTempDir("missing-pkg"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "hello.recipe"), + "return { name = 'hello', version = '1.0.0' }"); + + // Create a fake that exits 0 but creates no package.lua + auto fakePath = buildPath(tmp, "fake-nopkg"); + string noPkgScript = "#!/bin/bash\necho 'done' >&2\nexit 0\n"; + () @trusted { + write(fakePath, noPkgScript); + chmod(toStringz(fakePath), octal!755); + }(); + + auto outDir = buildPath(tmp, "output"); + + Config cfg; + cfg.zetaToolchainPath = fakePath; + + try { + runMakepkg(buildPath(tmp, "hello.recipe"), outDir, 4, false, cfg); + assert(false, "Expected BuildException for missing package.lua"); + } catch (BuildException e) { + assert(e.msg.indexOf("package.lua not found") >= 0, + "Expected 'package.lua not found', got: " ~ e.msg); + } + } + + // ── Test (7): pkgNameFromPath edge cases ──────────────────────────────── + @safe unittest { + assert(pkgNameFromPath("hello.recipe") == "hello"); + assert(pkgNameFromPath("/path/to/pkg.recipe") == "pkg"); + assert(pkgNameFromPath("a.b.recipe") == "a.b"); // last .recipe + assert(pkgNameFromPath("norecipe") == "norecipe"); // no dot at all + assert(pkgNameFromPath("/tmp/.config/build.sh") == "build"); + } +}