From a42001b9d8e4ce7855bb5ed86811c182d79cb284 Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Fri, 7 Aug 2026 15:11:00 -0400 Subject: [PATCH] feat: --all batches and -w for multi building. --all builds all the files in a specified directory. -w/--workers specifies how many workers to send out to build multiple packages at once. -j specifies how many jobs each worker has. defaults to one. --- toolchain/builder/zeta-makepkg | 120 ++++++------------- toolchain/lib/build.lua | 208 +++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 85 deletions(-) create mode 100644 toolchain/lib/build.lua diff --git a/toolchain/builder/zeta-makepkg b/toolchain/builder/zeta-makepkg index fa15e36..690f588 100755 --- a/toolchain/builder/zeta-makepkg +++ b/toolchain/builder/zeta-makepkg @@ -44,12 +44,10 @@ package.path = here .. "/../lib/?.lua;" .. here .. "/../lib/zeta-toolchain/?.lua;" .. package.path -local recipe = require("recipe") -local builder = require("builder") -local packager = require("packager") -local indexer = require("indexer") local path = require("path") local log = require("log") +local build = require("build") +local indexer = require("indexer") --------------------------------------------------------------------------- -- Help @@ -60,10 +58,12 @@ zeta-makepkg — build Zeta binary packages from declarative .recipe files. Usage: zeta-makepkg [flags] + zeta-makepkg --all [flags] zeta-makepkg --index [flags] Commands: Build a package from a .recipe file + --all Build all .recipe files found in --index Regenerate packages/index.lua from existing package.lua files Flags: @@ -72,6 +72,7 @@ Flags: --repo Base URL for generated package.lua manifests (default: https://raw.githubusercontent.com/gretagen/...) -j, --jobs Number of parallel build jobs (default: nproc) + -w, --workers Number of parallel package builds for --all (default: 1) --no-index Don't update packages/index.lua after building --keep-work Don't remove the build work directory --force Overwrite existing package tarball @@ -84,6 +85,7 @@ Cache: Examples: zeta-makepkg mypkg.recipe zeta-makepkg mypkg.recipe --output ~/zeta-packages --force + zeta-makepkg --all recipes/ --output ~/zeta-packages zeta-makepkg --index --output ~/zeta-packages ]] @@ -98,9 +100,11 @@ local function parse_cli(args) keep_work = false, force = false, index = false, + all = nil, help = false, recipe = nil, jobs = nil, -- nil = auto-detect (nproc) + workers = 1, -- parallel package builds (--all only) repo = "https://raw.githubusercontent.com/gretagen/zeta-packages/refs/heads/main", } @@ -111,6 +115,13 @@ local function parse_cli(args) opts.help = true elseif a == "--index" then opts.index = true + elseif a == "--all" then + i = i + 1 + if i > #args then + io.stderr:write("error: --all requires a directory argument\n") + return nil + end + opts.all = args[i] elseif a == "--output" then i = i + 1 if i > #args then @@ -158,6 +169,21 @@ local function parse_cli(args) return nil end opts.jobs = n + elseif a == "--workers" then + i = i + 1 + if i > #args then + io.stderr:write("error: --workers requires a number argument\n") + return nil + end + local n = tonumber(args[i]) + if not n or n < 1 then + io.stderr:write(("error: invalid workers value %q\n"):format(args[i])) + return nil + end + opts.workers = n + elseif a:match("^%-w%d+$") then + local n = tonumber(a:match("^%-w(%d+)$")) + opts.workers = n elseif a:match("^%-") then io.stderr:write(("error: unknown flag %s\n"):format(a)) return nil @@ -174,85 +200,7 @@ local function parse_cli(args) end --------------------------------------------------------------------------- --- Build pipeline ---------------------------------------------------------------------------- - -local function run_build(recipe_path, opts) - -- 1. Load recipe - local r, err = recipe.load(recipe_path) - if not r then - log.fatal(("recipe error: %s"):format(tostring(err))) - end - - -- 2. Set up output directory - local packages_dir = path.join(opts.output, "packages") - local pkg_dir = path.join(packages_dir, r.name) - - if path.exists(pkg_dir) and not opts.force then - log.fatal(("package %s already exists at %s (use --force to overwrite)"):format( - r.name, pkg_dir)) - end - path.mkdir_p(pkg_dir) - - -- 3. Work directories - local stamp = os.date("%Y%m%d-%H%M%S") - local work_dir = path.join("/tmp/zeta-makepkg", r.name .. "-" .. stamp) - local stage_dir = path.join(work_dir, "stage") - path.mkdir_p(stage_dir) - - log.info(("work directory: %s"):format(work_dir)) - - -- 4. Cache directory - local cache_dir = path.join(os.getenv("HOME") or "/tmp", ".cache", "zeta-makepkg", "sources") - - -- 5. Fetch source - local cache_file = builder.fetch_source(r, cache_dir) - - -- 6. Extract source - local source_dir = builder.extract_source(r, cache_file, work_dir) - - -- 7. Build - builder.run_build(r, source_dir, stage_dir, { jobs = opts.jobs }) - - -- 8. Test (optional) - if r.test then - builder.run_test(r.test, stage_dir) - else - log.info("no test command — skipping verification") - end - - -- 9. Package - local tarball_path = path.join(pkg_dir, r.name .. "-" .. r.version .. ".tar.gz") - local tarball_sha256 = packager.create_tarball(stage_dir, tarball_path) - - -- 10. Write package.lua manifest - packager.write_manifest(r, tarball_sha256, pkg_dir, opts.repo) - - -- 11. Update index - if not opts.no_index then - indexer.generate(packages_dir) - else - log.info("--no-index: skipping index update") - end - - -- 12. Cleanup - if not opts.keep_work then - path.run("rm -rf " .. path.quote(work_dir)) - else - log.info(("--keep-work: work directory preserved at %s"):format(work_dir)) - end - - log.ok(("%s-%s built successfully"):format(r.name, r.version)) - log.info(("output: %s"):format(pkg_dir)) -end - -local function run_index(opts) - local packages_dir = path.join(opts.output, "packages") - indexer.generate(packages_dir) -end - ---------------------------------------------------------------------------- --- Main +-- Dispatch --------------------------------------------------------------------------- local args = {} @@ -270,9 +218,11 @@ if opts.help then end if opts.index then - run_index(opts) + indexer.generate(path.join(opts.output, "packages")) +elseif opts.all then + build.all(opts.all, opts, arg[0]) elseif opts.recipe then - local ok, err = pcall(run_build, opts.recipe, opts) + local ok, err = pcall(build.single, opts.recipe, opts) if not ok then log.fatal(tostring(err)) end diff --git a/toolchain/lib/build.lua b/toolchain/lib/build.lua new file mode 100644 index 0000000..e402f40 --- /dev/null +++ b/toolchain/lib/build.lua @@ -0,0 +1,208 @@ +-- build.lua -- single-recipe pipeline and batch dispatcher. +-- +-- build_single(recipe_path, opts) → runs the full pipeline for one recipe +-- build_all(dir, opts, script_path) → batch build (sequential or parallel) + +local build = {} + +local path = require("path") +local log = require("log") +local recipe = require("recipe") +local builder = require("builder") +local packager = require("packager") +local indexer = require("indexer") + +--------------------------------------------------------------------------- +-- Single recipe pipeline +--------------------------------------------------------------------------- + +function build.single(recipe_path, opts) + local r, err = recipe.load(recipe_path) + if not r then error(("recipe error: %s"):format(tostring(err)), 0) end + + local packages_dir = path.join(opts.output, "packages") + local pkg_dir = path.join(packages_dir, r.name) + + if path.exists(pkg_dir) and not opts.force then + error(("package %s already exists at %s (use --force to overwrite)"):format( + r.name, pkg_dir), 0) + end + path.mkdir_p(pkg_dir) + + local stamp = os.date("%Y%m%d-%H%M%S") + local work_dir = path.join("/tmp/zeta-makepkg", r.name .. "-" .. stamp) + local stage_dir = path.join(work_dir, "stage") + path.mkdir_p(stage_dir) + + log.info(("work directory: %s"):format(work_dir)) + + local cache_dir = path.join(os.getenv("HOME") or "/tmp", ".cache", "zeta-makepkg", "sources") + local cache_file = builder.fetch_source(r, cache_dir) + local source_dir = builder.extract_source(r, cache_file, work_dir) + + builder.run_build(r, source_dir, stage_dir, { jobs = opts.jobs }) + + if r.test then + builder.run_test(r.test, stage_dir) + else + log.info("no test command — skipping verification") + end + + local tarball_path = path.join(pkg_dir, r.name .. "-" .. r.version .. ".tar.gz") + local sha256 = packager.create_tarball(stage_dir, tarball_path) + packager.write_manifest(r, sha256, pkg_dir, opts.repo) + + if not opts.no_index then + indexer.generate(packages_dir) + else + log.info("--no-index: skipping index update") + end + + if not opts.keep_work then + path.run("rm -rf " .. path.quote(work_dir)) + else + log.info(("--keep-work: work directory preserved at %s"):format(work_dir)) + end + + log.ok(("%s-%s built successfully"):format(r.name, r.version)) + log.info(("output: %s"):format(pkg_dir)) +end + +--------------------------------------------------------------------------- +-- Collect recipes from a directory +--------------------------------------------------------------------------- + +local function collect_recipes(dir) + local recipes = {} + local f = io.popen("find " .. path.quote(dir) + .. " -maxdepth 1 -name '*.recipe' -type f 2>/dev/null | sort") + if f then + for line in f:lines() do + if line ~= "" then recipes[#recipes + 1] = line end + end + f:close() + end + return recipes +end + +--------------------------------------------------------------------------- +-- Batch build +--------------------------------------------------------------------------- + +function build.all(dir, opts, script_path) + if not path.exists(dir) then + error(("directory not found: %s"):format(dir), 0) + end + + log.step(("scanning %s for recipes"):format(dir)) + + local recipes = collect_recipes(dir) + if #recipes == 0 then + error(("no .recipe files found in %s"):format(dir), 0) + end + + log.info(("found %d recipe(s)"):format(#recipes)) + + local batch_opts = { + output = opts.output, + no_index = true, + keep_work = opts.keep_work, + force = opts.force, + jobs = opts.jobs, + repo = opts.repo, + } + + local failed = {} + local workers = opts.workers or 1 + + if workers <= 1 then + for _, recipe_path in ipairs(recipes) do + io.write("\n" .. string.rep("─", 48) .. "\n") + log.step(("building %s"):format(path.basename(recipe_path))) + local ok, err = pcall(build.single, recipe_path, batch_opts) + if not ok then + log.error(tostring(err)) + failed[#failed + 1] = { path = recipe_path, err = tostring(err) } + end + end + else + local task_dir = "/tmp/zeta-makepkg-workers" + path.run("rm -rf " .. path.quote(task_dir)) + path.mkdir_p(task_dir) + + local jobs_flag = opts.jobs and (" -j" .. opts.jobs) or "" + local running = 0 + + local function count_alive() + local n = 0 + for i = 1, #recipes do + if not path.exists(task_dir .. "/" .. i .. ".done") then n = n + 1 end + end + return n + end + + for i, recipe_path in ipairs(recipes) do + while running >= workers do + os.execute("sleep 0.5") + running = count_alive() + end + + local done_file = task_dir .. "/" .. i .. ".done" + local log_file = task_dir .. "/" .. i .. ".log" + local cmd = string.format( + "(%s %s %s --output %s --no-index --force --repo %s > %s 2>&1; echo $? > %s) &", + path.quote(script_path), jobs_flag, path.quote(recipe_path), + path.quote(opts.output), path.quote(opts.repo), + path.quote(log_file), path.quote(done_file)) + + os.execute(cmd) + running = running + 1 + log.info(("[%d/%d] spawned %s"):format(i, #recipes, path.basename(recipe_path))) + end + + log.step("waiting for workers...") + while count_alive() > 0 do + os.execute("sleep 0.5") + end + + for i = 1, #recipes do + local done_file = task_dir .. "/" .. i .. ".done" + local f = io.open(done_file, "r") + local exit_code = f and tonumber(f:read("*l")) or 1 + if f then f:close() end + if exit_code ~= 0 then + local err_msg = ("exit code %d"):format(exit_code) + local lf = io.open(task_dir .. "/" .. i .. ".log", "r") + if lf then + for line in lf:lines() do + if line:match("error") then err_msg = line end + end + lf:close() + end + failed[#failed + 1] = { path = recipes[i], err = err_msg } + else + log.ok(path.basename(recipes[i])) + end + end + + path.run("rm -rf " .. path.quote(task_dir)) + end + + io.write("\n" .. string.rep("─", 48) .. "\n") + if not opts.no_index then + indexer.generate(path.join(opts.output, "packages")) + end + + io.write("\n") + local built = #recipes - #failed + log.ok(("%d built, %d failed"):format(built, #failed)) + if #failed > 0 then + log.warn("failed recipes:") + for _, f in ipairs(failed) do + log.warn((" %s — %s"):format(path.basename(f.path), f.err)) + end + os.exit(1) + end +end + +return build