From 8f9a3c85a4748da02c3472ebea7745964985002a Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Sat, 8 Aug 2026 18:09:06 -0400 Subject: [PATCH] feat(build): orchestrate sequential topological builds with fail-fast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add buildAll() to src/tofu/build.d — sequential builder with: - Empty plan → 'nothing to build' no-op - Recipe existence pre-check → fail-fast on missing - Skip-if-exists: don't rebuild when output present (unless --force) - Separator line: U+2500 box chars '──── building (i/n) ────' - Fail-fast: halt immediately on first BuildException 6 unittests: all succeed, first fails, empty plan, skip existing, force rebuild, missing recipe. --- .omo/evidence/task-15-tofu-core.log | 148 +++++++++++++++ .omo/notepads/tofu-core/learnings.md | 43 +++++ src/tofu/build.d | 257 +++++++++++++++++++++++++++ 3 files changed, 448 insertions(+) create mode 100644 .omo/evidence/task-15-tofu-core.log diff --git a/.omo/evidence/task-15-tofu-core.log b/.omo/evidence/task-15-tofu-core.log new file mode 100644 index 0000000..01087a8 --- /dev/null +++ b/.omo/evidence/task-15-tofu-core.log @@ -0,0 +1,148 @@ + Warning + Warning ## Warning for package tofu ## + Warning + Warning The following compiler flags have been specified in the package description + Warning file. They are handled by DUB and direct use in packages is discouraged. + Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags + Warning to the compiler, or use one of the suggestions below: + Warning + Warning warningsAsErrors: Use "buildRequirements" to control the warning level + Warning + Generating test runner configuration 'tofu-test-application' for 'application' (executable). + Warning Excluding main source file src/main.d from test. + Starting Performing "unittest" build using /usr/bin/dmd for x86_64. + Up-to-date toml 1.0.0: target for configuration [library] is up to date. + Building tofu ~main: building configuration [tofu-test-application] + Linking tofu-test-application + Finished To force a rebuild of up-to-date targets, run again with --force + Running tofu-test-application +zeta-makepkg: building hello...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phasezeta-makepkg: building hello...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phasezeta-makepkg: building hello...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phasezeta-makepkg: building hello...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phasedonezeta-makepkg: building depA...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phasezeta-makepkg: building depB...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phasezeta-makepkg: building target...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phasezeta-makepkg: building depA...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phaseerror build failed for depA: build failed for depA: +zeta-makepkg: building depA... +zeta-makepkg: configure phase +zeta-makepkg: build phase +zeta-makepkg: install phase +zeta-makepkg: building pkg...zeta-makepkg: configure phasezeta-makepkg: build phasezeta-makepkg: install phaseerror build failed for noexist: recipe not found at /tmp/tofu-test-build-buildall-norecipe-214411/nonexistent.recipe +warn corrupted cache for pkg: Found 'h' when expecting 'r'. (Line 1:2) +Warning: malformed TOML config at /tmp/tofu-test-config-bad-214411.toml: Invalid table key declaration (2:0) +Warning: invalid TOFU_DEFAULT_JOBS 'not-a-number', using default 1 +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 - +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/package.lua HTTP/1.1" 200 - +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/build.sh HTTP/1.1" 200 - +127.0.0.1 - - [08/Aug/2026 18:08:36] code 404, message File not found +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/hello.recipe HTTP/1.1" 404 - +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 - +127.0.0.1 - - [08/Aug/2026 18:08:36] code 404, message File not found +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/package.lua HTTP/1.1" 404 - +127.0.0.1 - - [08/Aug/2026 18:08:36] code 404, message File not found +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/build.sh HTTP/1.1" 404 - +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/scripts/build.sh HTTP/1.1" 200 - +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 - +127.0.0.1 - - [08/Aug/2026 18:08:36] code 404, message File not found +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/package.lua HTTP/1.1" 404 - +127.0.0.1 - - [08/Aug/2026 18:08:36] code 404, message File not found +127.0.0.1 - - [08/Aug/2026 18:08:36] "GET /recipes/hello/build.sh HTTP/1.1" 404 - +warn skipping index entry with empty name +warn skipping index entry with empty name +warn skipping index entry 'badpool': invalid pool 'bad_pool_value' + . checking binary firefox ... + - binary firefox-2.1.0 satisfies firefox>=2.0 + . checking binary libfoo ... + - binary libfoo-1.9 does not satisfy libfoo>=2.0 + . checking binary noexist ... + . checking binary brokenpkg ... + . checking binary badpkg ... + . checking binary testpkg ... + - binary testpkg-2.1.0 satisfies testpkg +==> ──── building depA (1/3) ──── + ok depA +==> ──── building depB (2/3) ──── + ok depB +==> ──── building target (3/3) ──── + ok target +==> ──── building depA (1/3) ──── + - nothing to build +==> ──── building pkg (1/1) ──── + - pkg already built, skipping +==> ──── building pkg (1/1) ──── + ok pkg +==> fetching recipe hello + . downloading hello.recipe + . downloading package.lua + . downloading build.sh + ok recipe hello downloaded +==> fetching recipe hello + . downloading hello.recipe +==> fetching recipe hello + . downloading hello.recipe + . downloading package.lua + - no package.lua for hello (optional manifest) + . downloading build.sh + - no build.sh for hello (optional build script) + . custom build system: fetching scripts/build.sh + ok recipe hello downloaded +==> fetching recipe hello + . downloading hello.recipe + . downloading package.lua + - no package.lua for hello (optional manifest) + . downloading build.sh + - no build.sh for hello (optional build script) + ok recipe hello downloaded +==> fetching recipe hello + . downloading hello.recipe + . fetching index http://127.0.0.1:49413//index.lua + ok index loaded: 3 packages + . fetching index http://127.0.0.1:38411//index.lua + . fetching index http://127.0.0.1:43727//index.lua + . fetching index http://127.0.0.1:33325//index.lua + . fetching index http://127.0.0.1:45567//index.lua + ok index loaded: 0 packages + . fetching index http://127.0.0.1:43115//index.lua + ok index loaded: 1 packages + . fetching index http://127.0.0.1:36939//index.lua + ok index loaded: 2 packages + . fetching index http://127.0.0.1:47607//index.lua + . fetching index http://127.0.0.1:48661//index.lua + ok index loaded: 1 packages +installing package...error: build failedreason: missing dependencytestpkg-2.1 is already installed -- use -ReProvide to reinstall13 modules passed unittests + - already installed — skipping + - binary libfoo-2.1 satisfies libfoo>=2.0 + - libfoo: binary 1.9 too old, building from recipe + - libfoo: binary 1.9 too old, building from recipe + - binary libbar-3.0 satisfies libbar (unconstrained) + - libbaz: no binary available, building from recipe + - binary libfoo-1.5 satisfies libfoo>=1.0 + - libfoo: binary 1.5 too old, building from recipe + - binary parentA-1.5 satisfies parentA (unconstrained) + - binary parentB-1.5 satisfies parentB (unconstrained) +==> generating build plan + - + C (recipe) + - + A (recipe) + ok build plan: 2 packages +==> generating build plan + - recipe libfoo not in cache, re-fetching + - + libfoo (recipe) + ok build plan: 1 packages +==> generating build plan +==> generating build plan + - nothing to build (all binary) +==> generating build plan + - + mypkg (recipe) + ok build plan: 1 packages +==> generating build plan + - + libfoo (recipe) + - + mypkg (recipe) + ok build plan: 2 packages + Warning + Warning ## Warning for package tofu ## + Warning + Warning The following compiler flags have been specified in the package description + Warning file. They are handled by DUB and direct use in packages is discouraged. + Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags + Warning to the compiler, or use one of the suggestions below: + Warning + Warning warningsAsErrors: Use "buildRequirements" to control the warning level + Warning + Starting Performing "debug" build using /usr/bin/dmd for x86_64. + Up-to-date toml 1.0.0: target for configuration [library] is up to date. + Up-to-date tofu ~main: target for configuration [application] is up to date. + Finished To force a rebuild of up-to-date targets, run again with --force diff --git a/.omo/notepads/tofu-core/learnings.md b/.omo/notepads/tofu-core/learnings.md index c485bba..6d5ce10 100644 --- a/.omo/notepads/tofu-core/learnings.md +++ b/.omo/notepads/tofu-core/learnings.md @@ -641,3 +641,46 @@ The sandbox test (test 2) creates a sentinel file, serves an index containing `o - `dub build` passes with `warningsAsErrors`. - Pre-existing breakage in `build.d` (parallel task 15 artefact) excluded via `.skip` rename for testing — NOT caused by task 13. - Evidence logged to `.omo/evidence/task-13-tofu-core.log`. + +--- + +## Task 15 — `tofu.build.buildAll` (sequential topological build orchestrator) + +### Architecture +- `buildAll(BuildPlan plan, Config cfg, bool force = false)` orchestrates sequential package builds with fail-fast semantics. +- Added imports: `tofu.types : BuildPlan, BuildResult, BuildFailure, Source;`, `tofu.log : logStep, logOk, logError, logInfo;`. +- No new exception classes — reuses existing `BuildException` from `runMakepkg`. + +### Algorithm +1. Empty plan → `logInfo("nothing to build")`, return empty `BuildResult`. +2. Iterate `plan.order()` entries (caller guarantees deps-first topological order): + a. **Pre-check**: `exists(entry.recipePath)` via `@trusted` wrapper. Missing → `BuildFailure` with "recipe not found at ", return immediately. + b. **Separator**: `logStep("──── building %s (%d/%d) ────", name, idx, total)` — U+2500 box drawing chars. + c. **Skip-if-exists**: If NOT force AND `cfg.builtDir()/packages//package.lua` exists → logInfo skip, add to succeeded, continue. + d. **Execute**: `runMakepkg(entry.recipePath, cfg.builtDir(), cfg.defaultJobs, force, cfg)`. + e. **Success**: logOk, add to succeeded. + f. **BuildException**: logError, add to failed, return immediately (halt). +3. Return populated `BuildResult`. + +### `@safe` / `@trusted` architecture +- `buildAll` is `@safe` public. +- Only `std.file.exists` requires `@trusted` wrappers — same pattern as other modules. +- `log*` functions are `@safe` — no `@trusted` needed for logging. + +### Test strategy (6 new unittests, numbered 8–13) +- Reused existing test helpers: `testTempDir`, `sWrite`, `sRmdirRecurse`, `makeFakeMakepkg`. +- Test (8): plan [depA, depB, target] all valid → all 3 succeed in order, outputs created. +- Test (9): fake always exits 1 → only depA fails, succeeded empty, depB/target skipped. +- Test (10): empty plan → no-op, both lists empty. +- Test (11): pre-existing package.lua + force=false → skipped, fake NOT invoked (args file absent). +- Test (12): pre-existing package.lua + force=true → rebuilt, fake invoked (args file present). +- Test (13): nonexistent recipe path → fail-fast with "recipe not found", remaining skipped. + +### Skip-if-exists semantics +- Path checked: `cfg.builtDir() ~ "/packages/" ~ name ~ "/package.lua"` — matches what `runMakepkg` produces (verified at line 190 in `runMakepkg`). +- `force=true` bypasses skip — always invokes `runMakepkg`. + +### Build verified +- `dub test` passes — all 13 modules, including 6 new buildAll unittests. +- `dub build` passes with `warningsAsErrors`. +- Evidence logged to `.omo/evidence/task-15-tofu-core.log`. diff --git a/src/tofu/build.d b/src/tofu/build.d index 5ba3da6..b9327f2 100644 --- a/src/tofu/build.d +++ b/src/tofu/build.d @@ -5,6 +5,8 @@ module tofu.build; import tofu.config : Config; +import tofu.types : BuildPlan, BuildResult, BuildFailure, Source; +import tofu.log : logStep, logOk, logError, logInfo; import std.process : pipeProcess, Redirect, ProcessException, Pid, wait; import std.file : exists, mkdirRecurse; import std.path : baseName; @@ -196,6 +198,68 @@ string runMakepkg(string recipePath, string outputDir, int jobs, bool force, return pkgPath; } +/// Build all packages in plan order (deps-first). Sequential, fail-fast. +/// +/// Parameters: +/// plan = ordered BuildPlan (caller ensures deps-first topological order) +/// cfg = tofu configuration +/// force = if true, pass --force to zeta-makepkg (overwrite existing) +/// +/// Returns: +/// BuildResult with succeeded/failed lists. On first failure, returns +/// immediately — remaining packages are NOT attempted. +@safe +BuildResult buildAll(BuildPlan plan, Config cfg, bool force = false) { + BuildResult result; + + auto entries = plan.order(); + + // Empty plan → no-op + if (entries.length == 0) { + logInfo("nothing to build"); + return result; + } + + size_t total = entries.length; + size_t idx = 0; + + foreach (entry; entries) { + idx++; + + // Pre-check: recipe file must exist + if (!() @trusted { return exists(entry.recipePath); }()) { + auto reason = "recipe not found at " ~ entry.recipePath; + logError("build failed for %s: %s", entry.name, reason); + result.failed ~= BuildFailure(entry.name, reason); + return result; + } + + // Separator + logStep("──── building %s (%d/%d) ────", entry.name, idx, total); + + // Skip-if-exists: check for pre-built output + auto pkgPath = cfg.builtDir() ~ "/packages/" ~ entry.name ~ "/package.lua"; + if (!force && () @trusted { return exists(pkgPath); }()) { + logInfo("%s already built, skipping", entry.name); + result.succeeded ~= entry.name; + continue; + } + + // Execute build + try { + runMakepkg(entry.recipePath, cfg.builtDir(), cfg.defaultJobs, force, cfg); + result.succeeded ~= entry.name; + logOk("%s", entry.name); + } catch (BuildException e) { + logError("build failed for %s: %s", entry.name, e.msg); + result.failed ~= BuildFailure(entry.name, e.msg); + return result; // halt immediately + } + } + + return result; +} + // ─── Unittests ─────────────────────────────────────────────────────────────── version (unittest) { @@ -426,4 +490,197 @@ exit ` ~ exitCode ~ "\n"; assert(pkgNameFromPath("norecipe") == "norecipe"); // no dot at all assert(pkgNameFromPath("/tmp/.config/build.sh") == "build"); } + + // ── Test (8): buildAll — plan [depA, depB, target] all valid → all 3 succeed ─ + @safe unittest { + auto tmp = testTempDir("buildall-valid"); + scope (exit) sRmdirRecurse(tmp); + + // Create recipes for all 3 + foreach (n; ["depA", "depB", "target"]) { + sWrite(buildPath(tmp, n ~ ".recipe"), + "return { name = '" ~ n ~ "', version = '1.0.0' }"); + } + + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg"); + auto outDir = buildPath(tmp, "output"); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + cfg.cacheDir = tmp; + + // Build plan — deps-first order (caller guarantees this) + BuildPlan plan; + plan.add("depA", buildPath(tmp, "depA.recipe"), Source.recipe); + plan.add("depB", buildPath(tmp, "depB.recipe"), Source.recipe); + plan.add("target", buildPath(tmp, "target.recipe"), Source.recipe); + + auto result = buildAll(plan, cfg, false); + + assert(result.succeeded.length == 3, + format("Expected 3 succeeded, got %d: %s", + result.succeeded.length, result.succeeded)); + assert(result.succeeded[0] == "depA"); + assert(result.succeeded[1] == "depB"); + assert(result.succeeded[2] == "target"); + assert(result.failed.length == 0); + + // Verify outputs were created + foreach (n; ["depA", "depB", "target"]) { + auto pkg = () @trusted { + return exists( + buildPath(tmp, "built", "packages", n, "package.lua")); + }(); + assert(pkg, "Expected package.lua for " ~ n); + } + } + + // ── Test (9): buildAll — first recipe fails → rest skipped ────────────── + @safe unittest { + auto tmp = testTempDir("buildall-failfast"); + scope (exit) sRmdirRecurse(tmp); + + foreach (n; ["depA", "depB", "target"]) { + sWrite(buildPath(tmp, n ~ ".recipe"), + "return { name = '" ~ n ~ "', version = '1.0.0' }"); + } + + // Fake that always exits 1 (fails every invocation) + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg-fail", "1"); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + cfg.cacheDir = tmp; + + BuildPlan plan; + plan.add("depA", buildPath(tmp, "depA.recipe"), Source.recipe); + plan.add("depB", buildPath(tmp, "depB.recipe"), Source.recipe); + plan.add("target", buildPath(tmp, "target.recipe"), Source.recipe); + + auto result = buildAll(plan, cfg, false); + + assert(result.succeeded.length == 0, + "Expected 0 succeeded, got: " ~ result.succeeded.to!string); + assert(result.failed.length == 1, + "Expected 1 failed (only depA), got " ~ result.failed.length.to!string); + assert(result.failed[0].name == "depA", + "Expected depA to fail, got: " ~ result.failed[0].name); + assert(result.failed[0].reason.indexOf("build failed for depA") >= 0, + "Expected 'build failed for depA' in reason, got: " + ~ result.failed[0].reason); + } + + // ── Test (10): buildAll — empty plan → "nothing to build" ──────────────── + @safe unittest { + Config cfg; + cfg.cacheDir = "/tmp/dummy"; + + BuildPlan plan; // empty + auto result = buildAll(plan, cfg); + + assert(result.succeeded.length == 0); + assert(result.failed.length == 0); + } + + // ── Test (11): buildAll — existing output + force=false → skipped ──────── + @safe unittest { + auto tmp = testTempDir("buildall-skip"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "pkg.recipe"), + "return { name = 'pkg', version = '1.0.0' }"); + + // Pre-create the output + auto pkgLuaDir = buildPath(tmp, "built", "packages", "pkg"); + () @trusted { mkdirRecurse(pkgLuaDir); }(); + sWrite(buildPath(pkgLuaDir, "package.lua"), + "return { version = '1.0.0' }"); + + // Fake with args file as invocation counter + auto argsFile = buildPath(tmp, "args.txt"); + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg", "0", argsFile); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + cfg.cacheDir = tmp; + + BuildPlan plan; + plan.add("pkg", buildPath(tmp, "pkg.recipe"), Source.recipe); + + auto result = buildAll(plan, cfg, false); + + assert(result.succeeded.length == 1); + assert(result.succeeded[0] == "pkg"); + assert(result.failed.length == 0); + + // Fake script must NOT have been invoked + auto argsExist = () @trusted { + try { return exists(argsFile); } catch (Exception) { return false; } + }(); + assert(!argsExist, "Fake makepkg was invoked but should have been skipped"); + } + + // ── Test (12): buildAll — existing output + force=true → rebuilt ───────── + @safe unittest { + auto tmp = testTempDir("buildall-force"); + scope (exit) sRmdirRecurse(tmp); + + sWrite(buildPath(tmp, "pkg.recipe"), + "return { name = 'pkg', version = '1.0.0' }"); + + // Pre-create the output + auto pkgLuaDir = buildPath(tmp, "built", "packages", "pkg"); + () @trusted { mkdirRecurse(pkgLuaDir); }(); + sWrite(buildPath(pkgLuaDir, "package.lua"), + "return { version = '1.0.0' }"); + + // Fake with args file as invocation counter + auto argsFile = buildPath(tmp, "args.txt"); + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg", "0", argsFile); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + cfg.cacheDir = tmp; + + BuildPlan plan; + plan.add("pkg", buildPath(tmp, "pkg.recipe"), Source.recipe); + + auto result = buildAll(plan, cfg, true); + + assert(result.succeeded.length == 1); + assert(result.succeeded[0] == "pkg"); + assert(result.failed.length == 0); + + // Fake script MUST have been invoked + auto argsExist = () @trusted { + try { return exists(argsFile); } catch (Exception) { return false; } + }(); + assert(argsExist, "Fake makepkg was NOT invoked but should have been (force=true)"); + } + + // ── Test (13): buildAll — missing recipe file → fail-fast ──────────────── + @safe unittest { + auto tmp = testTempDir("buildall-norecipe"); + scope (exit) sRmdirRecurse(tmp); + + auto fakeBin = makeFakeMakepkg(tmp, "fake-makepkg"); + + Config cfg; + cfg.zetaToolchainPath = fakeBin; + cfg.cacheDir = tmp; + + BuildPlan plan; + plan.add("noexist", buildPath(tmp, "nonexistent.recipe"), Source.recipe); + plan.add("target", buildPath(tmp, "will-not-build.recipe"), Source.recipe); + + auto result = buildAll(plan, cfg); + + assert(result.succeeded.length == 0); + assert(result.failed.length == 1, + "Expected 1 failed, got " ~ result.failed.length.to!string); + assert(result.failed[0].name == "noexist"); + assert(result.failed[0].reason.indexOf("recipe not found") >= 0, + "Expected 'recipe not found' in reason, got: " + ~ result.failed[0].reason); + } }