feat: --follow for deps and a custom example

--follow now follows the deptree of the definition
--follow is incompatible with --all
build_system = "custom" example with build.sh

toolchain/builder/zeta-makepkg

@@ -44,12 +44,10 @@ package.path = here .. "/../lib/?.lua;"
   .. here .. "/../lib/zeta-toolchain/?.lua;"
   .. package.path
 
-local recipe  = require("recipe")
-local builder = require("builder")
-local packager = require("packager")
-local indexer = require("indexer")
 local path    = require("path")
 local log     = require("log")
+local build   = require("build")
+local indexer = require("indexer")
 
 ---------------------------------------------------------------------------
 -- Help
@@ -60,10 +58,12 @@ zeta-makepkg — build Zeta binary packages from declarative .recipe files.
 
 Usage:
   zeta-makepkg <recipe> [flags]
+  zeta-makepkg --all <dir> [flags]
   zeta-makepkg --index [flags]
 
 Commands:
   <recipe>           Build a package from a .recipe file
+  --all <dir>        Build all .recipe files found in <dir>
   --index            Regenerate packages/index.lua from existing package.lua files
 
 Flags:
@@ -72,8 +72,11 @@ Flags:
   --repo <url>       Base URL for generated package.lua manifests
                      (default: https://raw.githubusercontent.com/gretagen/...)
   -j, --jobs <N>    Number of parallel build jobs (default: nproc)
+  -w, --workers <N> Number of parallel package builds for --all (default: 1)
   --no-index         Don't update packages/index.lua after building
   --keep-work        Don't remove the build work directory
+  --follow          Build all dependencies first (bottom-up), then the package.
+                     Incompatible with --all.
   --force            Overwrite existing package tarball
   --help             Show this help
 
@@ -84,6 +87,7 @@ Cache:
 Examples:
   zeta-makepkg mypkg.recipe
   zeta-makepkg mypkg.recipe --output ~/zeta-packages --force
+  zeta-makepkg --all recipes/ --output ~/zeta-packages
   zeta-makepkg --index --output ~/zeta-packages
 ]]
 
@@ -98,9 +102,12 @@ local function parse_cli(args)
     keep_work = false,
     force     = false,
     index     = false,
+    all       = nil,
+    follow    = false,
     help      = false,
     recipe    = nil,
     jobs      = nil,  -- nil = auto-detect (nproc)
+    workers   = 1,    -- parallel package builds (--all only)
     repo      = "https://raw.githubusercontent.com/gretagen/zeta-packages/refs/heads/main",
   }
 
@@ -111,6 +118,13 @@ local function parse_cli(args)
       opts.help = true
     elseif a == "--index" then
       opts.index = true
+    elseif a == "--all" then
+      i = i + 1
+      if i > #args then
+        io.stderr:write("error: --all requires a directory argument\n")
+        return nil
+      end
+      opts.all = args[i]
     elseif a == "--output" then
       i = i + 1
       if i > #args then
@@ -131,6 +145,8 @@ local function parse_cli(args)
       opts.keep_work = true
     elseif a == "--force" then
       opts.force = true
+    elseif a == "--follow" then
+      opts.follow = true
     elseif a == "--jobs" then
       i = i + 1
       if i > #args then
@@ -158,6 +174,21 @@ local function parse_cli(args)
         return nil
       end
       opts.jobs = n
+    elseif a == "--workers" then
+      i = i + 1
+      if i > #args then
+        io.stderr:write("error: --workers requires a number argument\n")
+        return nil
+      end
+      local n = tonumber(args[i])
+      if not n or n < 1 then
+        io.stderr:write(("error: invalid workers value %q\n"):format(args[i]))
+        return nil
+      end
+      opts.workers = n
+    elseif a:match("^%-w%d+$") then
+      local n = tonumber(a:match("^%-w(%d+)$"))
+      opts.workers = n
     elseif a:match("^%-") then
       io.stderr:write(("error: unknown flag %s\n"):format(a))
       return nil
@@ -174,85 +205,7 @@ local function parse_cli(args)
 end
 
 ---------------------------------------------------------------------------
--- Build pipeline
----------------------------------------------------------------------------
-
-local function run_build(recipe_path, opts)
-  -- 1. Load recipe
-  local r, err = recipe.load(recipe_path)
-  if not r then
-    log.fatal(("recipe error: %s"):format(tostring(err)))
-  end
-
-  -- 2. Set up output directory
-  local packages_dir = path.join(opts.output, "packages")
-  local pkg_dir = path.join(packages_dir, r.name)
-
-  if path.exists(pkg_dir) and not opts.force then
-    log.fatal(("package %s already exists at %s (use --force to overwrite)"):format(
-      r.name, pkg_dir))
-  end
-  path.mkdir_p(pkg_dir)
-
-  -- 3. Work directories
-  local stamp = os.date("%Y%m%d-%H%M%S")
-  local work_dir = path.join("/tmp/zeta-makepkg", r.name .. "-" .. stamp)
-  local stage_dir = path.join(work_dir, "stage")
-  path.mkdir_p(stage_dir)
-
-  log.info(("work directory: %s"):format(work_dir))
-
-  -- 4. Cache directory
-  local cache_dir = path.join(os.getenv("HOME") or "/tmp", ".cache", "zeta-makepkg", "sources")
-
-  -- 5. Fetch source
-  local cache_file = builder.fetch_source(r, cache_dir)
-
-  -- 6. Extract source
-  local source_dir = builder.extract_source(r, cache_file, work_dir)
-
-  -- 7. Build
-  builder.run_build(r, source_dir, stage_dir, { jobs = opts.jobs })
-
-  -- 8. Test (optional)
-  if r.test then
-    builder.run_test(r.test, stage_dir)
-  else
-    log.info("no test command — skipping verification")
-  end
-
-  -- 9. Package
-  local tarball_path = path.join(pkg_dir, r.name .. "-" .. r.version .. ".tar.gz")
-  local tarball_sha256 = packager.create_tarball(stage_dir, tarball_path)
-
-  -- 10. Write package.lua manifest
-  packager.write_manifest(r, tarball_sha256, pkg_dir, opts.repo)
-
-  -- 11. Update index
-  if not opts.no_index then
-    indexer.generate(packages_dir)
-  else
-    log.info("--no-index: skipping index update")
-  end
-
-  -- 12. Cleanup
-  if not opts.keep_work then
-    path.run("rm -rf " .. path.quote(work_dir))
-  else
-    log.info(("--keep-work: work directory preserved at %s"):format(work_dir))
-  end
-
-  log.ok(("%s-%s built successfully"):format(r.name, r.version))
-  log.info(("output: %s"):format(pkg_dir))
-end
-
-local function run_index(opts)
-  local packages_dir = path.join(opts.output, "packages")
-  indexer.generate(packages_dir)
-end
-
----------------------------------------------------------------------------
--- Main
+-- Dispatch
 ---------------------------------------------------------------------------
 
 local args = {}
@@ -269,10 +222,17 @@ if opts.help then
   os.exit(0)
 end
 
+if opts.follow and opts.all then
+  io.stderr:write("error: --follow and --all are incompatible\n")
+  os.exit(1)
+end
+
 if opts.index then
-  run_index(opts)
+  indexer.generate(path.join(opts.output, "packages"))
+elseif opts.all then
+  build.all(opts.all, opts, arg[0])
 elseif opts.recipe then
-  local ok, err = pcall(run_build, opts.recipe, opts)
+  local ok, err = pcall(build.single, opts.recipe, opts)
   if not ok then
     log.fatal(tostring(err))
   end

toolchain/examples/custom-demo/build.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# build.sh — example custom build script for zeta-makepkg.
+#
+# Invoked from the extracted source directory.  The tool sets:
+#   $DESTDIR  → staging tree root (e.g. /tmp/zeta-makepkg/mytool-.../stage)
+#
+# This script must compile/assemble the software and install every output
+# file under $DESTDIR with standard FHS paths (usr/bin/, usr/lib/, etc.).
+#
+# For a real project you would replace the example below with:
+#   - Go:      go build -o "$DESTDIR/usr/bin/mytool" .
+#   - Rust:    cargo build --release && install -D target/release/mytool "$DESTDIR/usr/bin/mytool"
+#   - Zig:     zig build -Doptimize=ReleaseSafe --prefix /usr -Ddestdir="$DESTDIR"
+#   - Shell:   install -D mytool.sh "$DESTDIR/usr/bin/mytool"
+#   - Python:  install -D mytool.py "$DESTDIR/usr/bin/mytool"
+#   - Anything you can script.
+
+set -eu
+
+echo "==> Building mytool (custom)"
+
+# Standalone Go binary: init module, build, install
+go mod init mytool
+go build -o mytool .
+
+# Install into the staging tree
+install -Dm755 mytool "$DESTDIR/usr/bin/mytool"
+
+echo "==> Install complete"

toolchain/examples/custom-demo/custom.recipe

@@ -0,0 +1,23 @@
+-- custom.recipe — demonstrates build_system = "custom" with a build.sh script.
+--
+-- The custom build system runs `build_script` from inside the extracted
+-- source directory. The script receives $DESTDIR pointing at the staging
+-- tree where installed files must be placed.
+--
+-- To try this:
+--   1. Unpack the source tarball:  tar -xzf mytool-1.0.tar.gz
+--   2. Run the script manually:     cd mytool-1.0 && DESTDIR=/tmp/stage sh build.sh
+--   3. Inspect the staged output:   find /tmp/stage
+--   4. Build with makepkg:          zeta-makepkg custom.recipe
+
+return {
+  name         = "mytool",
+  version      = "1.0",
+  summary      = "Example custom-build package (static Go binary)",
+  url          = "mytool-1.0.tar.gz",
+  sha256       = nil,
+  deps         = {},
+  build_system = "custom",
+  build_script = "build.sh",
+  test         = "test -x ${DESTDIR}/usr/bin/mytool",
+}

toolchain/lib/build.lua

@@ -0,0 +1,284 @@
+-- build.lua -- single-recipe pipeline and batch dispatcher.
+--
+-- build_single(recipe_path, opts)  → runs the full pipeline for one recipe
+-- build_all(dir, opts, script_path) → batch build (sequential or parallel)
+
+local build = {}
+
+local path     = require("path")
+local log      = require("log")
+local recipe   = require("recipe")
+local builder  = require("builder")
+local packager = require("packager")
+local indexer  = require("indexer")
+
+---------------------------------------------------------------------------
+-- Dependency resolution (--follow)
+---------------------------------------------------------------------------
+
+-- Given a recipe directory and a dep spec (e.g. "ncurses" or "libfoo>=2.0"),
+-- return the path to <dep>.recipe if it exists, or nil.
+local function find_dep_recipe(recipe_dir, dep_spec)
+  local name = dep_spec:match("^([%w%._+%-]+)")
+  if not name then return nil end
+  local p = path.join(recipe_dir, name .. ".recipe")
+  if path.exists(p) then return p end
+  return nil
+end
+
+-- Recursively collect all dependency recipe paths, bottom-up (deps first).
+-- Returns an ordered list (no duplicates) and an in-progress set for cycle
+-- detection. Recipes for deps that don't have a .recipe file are silently
+-- skipped (assumed already installed).
+local function collect_deps(recipe_path, seen, order, in_progress)
+  if seen[recipe_path] then return end
+  if in_progress[recipe_path] then
+    error(("dependency cycle involving %s"):format(path.basename(recipe_path)), 0)
+  end
+
+  in_progress[recipe_path] = true
+
+  local r, err = recipe.load(recipe_path)
+  if not r then
+    in_progress[recipe_path] = nil
+    log.warn(("skipping %s: %s"):format(path.basename(recipe_path), tostring(err)))
+    return
+  end
+
+  local dir = path.dirname(recipe_path)
+  for _, dep in ipairs(r.deps) do
+    local dep_recipe = find_dep_recipe(dir, dep)
+    if dep_recipe then
+      collect_deps(dep_recipe, seen, order, in_progress)
+    end
+  end
+
+  in_progress[recipe_path] = nil
+  seen[recipe_path] = true
+  order[#order + 1] = recipe_path
+end
+
+-- Return an ordered list of recipe paths to build: all dependencies
+-- (bottom-up), then the target recipe itself.
+local function resolve_deps(recipe_path)
+  local seen = {}
+  local order = {}
+  local in_progress = {}
+  collect_deps(recipe_path, seen, order, in_progress)
+  return order
+end
+
+---------------------------------------------------------------------------
+-- Single recipe pipeline
+---------------------------------------------------------------------------
+
+function build.single(recipe_path, opts)
+  -- If --follow, build dependencies first (bottom-up order, already sorted).
+  if opts.follow then
+    log.step("resolving dependencies")
+    local ordered = resolve_deps(recipe_path)
+    local dep_count = #ordered - 1  -- last is the target itself
+    if dep_count > 0 then
+      log.info(("found %d dependenc%s"):format(dep_count, dep_count == 1 and "y" or "ies"))
+      for i = 1, dep_count do
+        io.write("\n" .. string.rep("─", 36) .. "\n")
+        log.step(("building dependency %s (%d/%d)"):format(
+          path.basename(ordered[i]), i, dep_count))
+        build.single(ordered[i], opts)
+      end
+      io.write("\n" .. string.rep("─", 36) .. "\n")
+    else
+      log.info("no unresolved dependencies")
+    end
+  end
+
+  local r, err = recipe.load(recipe_path)
+  if not r then error(("recipe error: %s"):format(tostring(err)), 0) end
+
+  local packages_dir = path.join(opts.output, "packages")
+  local pkg_dir = path.join(packages_dir, r.name)
+
+  if path.exists(pkg_dir) and not opts.force then
+    error(("package %s already exists at %s (use --force to overwrite)"):format(
+      r.name, pkg_dir), 0)
+  end
+  path.mkdir_p(pkg_dir)
+
+  local stamp = os.date("%Y%m%d-%H%M%S")
+  local work_dir = path.join("/tmp/zeta-makepkg", r.name .. "-" .. stamp)
+  local stage_dir = path.join(work_dir, "stage")
+  path.mkdir_p(stage_dir)
+
+  log.info(("work directory: %s"):format(work_dir))
+
+  local cache_dir = path.join(os.getenv("HOME") or "/tmp", ".cache", "zeta-makepkg", "sources")
+  local recipe_dir = path.dirname(recipe_path)
+  local cache_file = builder.fetch_source(r, cache_dir, recipe_dir)
+  local source_dir = builder.extract_source(r, cache_file, work_dir)
+
+  builder.run_build(r, source_dir, stage_dir, { jobs = opts.jobs })
+
+  if r.test then
+    builder.run_test(r.test, stage_dir)
+  else
+    log.info("no test command — skipping verification")
+  end
+
+  local tarball_path = path.join(pkg_dir, r.name .. "-" .. r.version .. ".tar.gz")
+  local sha256 = packager.create_tarball(stage_dir, tarball_path)
+  packager.write_manifest(r, sha256, pkg_dir, opts.repo)
+
+  if not opts.no_index then
+    indexer.generate(packages_dir)
+  else
+    log.info("--no-index: skipping index update")
+  end
+
+  if not opts.keep_work then
+    path.run("rm -rf " .. path.quote(work_dir))
+  else
+    log.info(("--keep-work: work directory preserved at %s"):format(work_dir))
+  end
+
+  log.ok(("%s-%s built successfully"):format(r.name, r.version))
+  log.info(("output: %s"):format(pkg_dir))
+end
+
+---------------------------------------------------------------------------
+-- Collect recipes from a directory
+---------------------------------------------------------------------------
+
+local function collect_recipes(dir)
+  local recipes = {}
+  local f = io.popen("find " .. path.quote(dir)
+    .. " -maxdepth 1 -name '*.recipe' -type f 2>/dev/null | sort")
+  if f then
+    for line in f:lines() do
+      if line ~= "" then recipes[#recipes + 1] = line end
+    end
+    f:close()
+  end
+  return recipes
+end
+
+---------------------------------------------------------------------------
+-- Batch build
+---------------------------------------------------------------------------
+
+function build.all(dir, opts, script_path)
+  if not path.exists(dir) then
+    error(("directory not found: %s"):format(dir), 0)
+  end
+
+  log.step(("scanning %s for recipes"):format(dir))
+
+  local recipes = collect_recipes(dir)
+  if #recipes == 0 then
+    error(("no .recipe files found in %s"):format(dir), 0)
+  end
+
+  log.info(("found %d recipe(s)"):format(#recipes))
+
+  local batch_opts = {
+    output    = opts.output,
+    no_index  = true,
+    keep_work = opts.keep_work,
+    force     = opts.force,
+    jobs      = opts.jobs,
+    repo      = opts.repo,
+  }
+
+  local failed = {}
+  local workers = opts.workers or 1
+
+  if workers <= 1 then
+    for _, recipe_path in ipairs(recipes) do
+      io.write("\n" .. string.rep("─", 48) .. "\n")
+      log.step(("building %s"):format(path.basename(recipe_path)))
+      local ok, err = pcall(build.single, recipe_path, batch_opts)
+      if not ok then
+        log.error(tostring(err))
+        failed[#failed + 1] = { path = recipe_path, err = tostring(err) }
+      end
+    end
+  else
+    local task_dir = "/tmp/zeta-makepkg-workers"
+    path.run("rm -rf " .. path.quote(task_dir))
+    path.mkdir_p(task_dir)
+
+    local jobs_flag = opts.jobs and (" -j" .. opts.jobs) or ""
+    local running = 0
+
+    local function count_alive()
+      local n = 0
+      for i = 1, #recipes do
+        if not path.exists(task_dir .. "/" .. i .. ".done") then n = n + 1 end
+      end
+      return n
+    end
+
+    for i, recipe_path in ipairs(recipes) do
+      while running >= workers do
+        os.execute("sleep 0.5")
+        running = count_alive()
+      end
+
+      local done_file = task_dir .. "/" .. i .. ".done"
+      local log_file  = task_dir .. "/" .. i .. ".log"
+      local cmd = string.format(
+        "(%s %s %s --output %s --no-index --force --repo %s > %s 2>&1; echo $? > %s) &",
+        path.quote(script_path), jobs_flag, path.quote(recipe_path),
+        path.quote(opts.output), path.quote(opts.repo),
+        path.quote(log_file), path.quote(done_file))
+
+      os.execute(cmd)
+      running = running + 1
+      log.info(("[%d/%d] spawned %s"):format(i, #recipes, path.basename(recipe_path)))
+    end
+
+    log.step("waiting for workers...")
+    while count_alive() > 0 do
+      os.execute("sleep 0.5")
+    end
+
+    for i = 1, #recipes do
+      local done_file = task_dir .. "/" .. i .. ".done"
+      local f = io.open(done_file, "r")
+      local exit_code = f and tonumber(f:read("*l")) or 1
+      if f then f:close() end
+      if exit_code ~= 0 then
+        local err_msg = ("exit code %d"):format(exit_code)
+        local lf = io.open(task_dir .. "/" .. i .. ".log", "r")
+        if lf then
+          for line in lf:lines() do
+            if line:match("error") then err_msg = line end
+          end
+          lf:close()
+        end
+        failed[#failed + 1] = { path = recipes[i], err = err_msg }
+      else
+        log.ok(path.basename(recipes[i]))
+      end
+    end
+
+    path.run("rm -rf " .. path.quote(task_dir))
+  end
+
+  io.write("\n" .. string.rep("─", 48) .. "\n")
+  if not opts.no_index then
+    indexer.generate(path.join(opts.output, "packages"))
+  end
+
+  io.write("\n")
+  local built = #recipes - #failed
+  log.ok(("%d built, %d failed"):format(built, #failed))
+  if #failed > 0 then
+    log.warn("failed recipes:")
+    for _, f in ipairs(failed) do
+      log.warn(("  %s — %s"):format(path.basename(f.path), f.err))
+    end
+    os.exit(1)
+  end
+end
+
+return build

toolchain/lib/builder.lua

@@ -104,7 +104,7 @@ end
 
 -- Fetch source into the cache directory. Returns the cached file path,
 -- or nil if the URL is a git repository (signals "use git clone").
-function builder.fetch_source(recipe, cache_dir)
+function builder.fetch_source(recipe, cache_dir, local_dir)
   local url = recipe.url
 
   -- Git repos are cloned, not cached as files.
@@ -141,7 +141,16 @@ function builder.fetch_source(recipe, cache_dir)
         error(("failed to copy %s"):format(url), 0)
       end
     else
-      error(("unrecognised url scheme: %s"):format(url), 0)
+      -- Bare filename: resolve relative to the recipe's directory.
+      if local_dir then
+        local src = path.join(local_dir, url)
+        log.step("copying " .. src)
+        if not path.run("cp " .. path.quote(src) .. " " .. path.quote(cache_file)) then
+          error(("failed to copy %s"):format(src), 0)
+        end
+      else
+        error(("unrecognised url scheme: %s"):format(url), 0)
+      end
     end
   else
     log.info("using cached source: " .. path.basename(cache_file))