feat: --all batches and -w for multi building.
--all builds all the files in a specified directory. -w/--workers specifies how many workers to send out to build multiple packages at once. -j specifies how many jobs each worker has. defaults to one.
This commit is contained in:
@@ -44,12 +44,10 @@ package.path = here .. "/../lib/?.lua;"
|
||||
.. here .. "/../lib/zeta-toolchain/?.lua;"
|
||||
.. package.path
|
||||
|
||||
local recipe = require("recipe")
|
||||
local builder = require("builder")
|
||||
local packager = require("packager")
|
||||
local indexer = require("indexer")
|
||||
local path = require("path")
|
||||
local log = require("log")
|
||||
local build = require("build")
|
||||
local indexer = require("indexer")
|
||||
|
||||
---------------------------------------------------------------------------
|
||||
-- Help
|
||||
@@ -60,10 +58,12 @@ zeta-makepkg — build Zeta binary packages from declarative .recipe files.
|
||||
|
||||
Usage:
|
||||
zeta-makepkg <recipe> [flags]
|
||||
zeta-makepkg --all <dir> [flags]
|
||||
zeta-makepkg --index [flags]
|
||||
|
||||
Commands:
|
||||
<recipe> Build a package from a .recipe file
|
||||
--all <dir> Build all .recipe files found in <dir>
|
||||
--index Regenerate packages/index.lua from existing package.lua files
|
||||
|
||||
Flags:
|
||||
@@ -72,6 +72,7 @@ Flags:
|
||||
--repo <url> Base URL for generated package.lua manifests
|
||||
(default: https://raw.githubusercontent.com/gretagen/...)
|
||||
-j, --jobs <N> Number of parallel build jobs (default: nproc)
|
||||
-w, --workers <N> Number of parallel package builds for --all (default: 1)
|
||||
--no-index Don't update packages/index.lua after building
|
||||
--keep-work Don't remove the build work directory
|
||||
--force Overwrite existing package tarball
|
||||
@@ -84,6 +85,7 @@ Cache:
|
||||
Examples:
|
||||
zeta-makepkg mypkg.recipe
|
||||
zeta-makepkg mypkg.recipe --output ~/zeta-packages --force
|
||||
zeta-makepkg --all recipes/ --output ~/zeta-packages
|
||||
zeta-makepkg --index --output ~/zeta-packages
|
||||
]]
|
||||
|
||||
@@ -98,9 +100,11 @@ local function parse_cli(args)
|
||||
keep_work = false,
|
||||
force = false,
|
||||
index = false,
|
||||
all = nil,
|
||||
help = false,
|
||||
recipe = nil,
|
||||
jobs = nil, -- nil = auto-detect (nproc)
|
||||
workers = 1, -- parallel package builds (--all only)
|
||||
repo = "https://raw.githubusercontent.com/gretagen/zeta-packages/refs/heads/main",
|
||||
}
|
||||
|
||||
@@ -111,6 +115,13 @@ local function parse_cli(args)
|
||||
opts.help = true
|
||||
elseif a == "--index" then
|
||||
opts.index = true
|
||||
elseif a == "--all" then
|
||||
i = i + 1
|
||||
if i > #args then
|
||||
io.stderr:write("error: --all requires a directory argument\n")
|
||||
return nil
|
||||
end
|
||||
opts.all = args[i]
|
||||
elseif a == "--output" then
|
||||
i = i + 1
|
||||
if i > #args then
|
||||
@@ -158,6 +169,21 @@ local function parse_cli(args)
|
||||
return nil
|
||||
end
|
||||
opts.jobs = n
|
||||
elseif a == "--workers" then
|
||||
i = i + 1
|
||||
if i > #args then
|
||||
io.stderr:write("error: --workers requires a number argument\n")
|
||||
return nil
|
||||
end
|
||||
local n = tonumber(args[i])
|
||||
if not n or n < 1 then
|
||||
io.stderr:write(("error: invalid workers value %q\n"):format(args[i]))
|
||||
return nil
|
||||
end
|
||||
opts.workers = n
|
||||
elseif a:match("^%-w%d+$") then
|
||||
local n = tonumber(a:match("^%-w(%d+)$"))
|
||||
opts.workers = n
|
||||
elseif a:match("^%-") then
|
||||
io.stderr:write(("error: unknown flag %s\n"):format(a))
|
||||
return nil
|
||||
@@ -174,85 +200,7 @@ local function parse_cli(args)
|
||||
end
|
||||
|
||||
---------------------------------------------------------------------------
|
||||
-- Build pipeline
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
local function run_build(recipe_path, opts)
|
||||
-- 1. Load recipe
|
||||
local r, err = recipe.load(recipe_path)
|
||||
if not r then
|
||||
log.fatal(("recipe error: %s"):format(tostring(err)))
|
||||
end
|
||||
|
||||
-- 2. Set up output directory
|
||||
local packages_dir = path.join(opts.output, "packages")
|
||||
local pkg_dir = path.join(packages_dir, r.name)
|
||||
|
||||
if path.exists(pkg_dir) and not opts.force then
|
||||
log.fatal(("package %s already exists at %s (use --force to overwrite)"):format(
|
||||
r.name, pkg_dir))
|
||||
end
|
||||
path.mkdir_p(pkg_dir)
|
||||
|
||||
-- 3. Work directories
|
||||
local stamp = os.date("%Y%m%d-%H%M%S")
|
||||
local work_dir = path.join("/tmp/zeta-makepkg", r.name .. "-" .. stamp)
|
||||
local stage_dir = path.join(work_dir, "stage")
|
||||
path.mkdir_p(stage_dir)
|
||||
|
||||
log.info(("work directory: %s"):format(work_dir))
|
||||
|
||||
-- 4. Cache directory
|
||||
local cache_dir = path.join(os.getenv("HOME") or "/tmp", ".cache", "zeta-makepkg", "sources")
|
||||
|
||||
-- 5. Fetch source
|
||||
local cache_file = builder.fetch_source(r, cache_dir)
|
||||
|
||||
-- 6. Extract source
|
||||
local source_dir = builder.extract_source(r, cache_file, work_dir)
|
||||
|
||||
-- 7. Build
|
||||
builder.run_build(r, source_dir, stage_dir, { jobs = opts.jobs })
|
||||
|
||||
-- 8. Test (optional)
|
||||
if r.test then
|
||||
builder.run_test(r.test, stage_dir)
|
||||
else
|
||||
log.info("no test command — skipping verification")
|
||||
end
|
||||
|
||||
-- 9. Package
|
||||
local tarball_path = path.join(pkg_dir, r.name .. "-" .. r.version .. ".tar.gz")
|
||||
local tarball_sha256 = packager.create_tarball(stage_dir, tarball_path)
|
||||
|
||||
-- 10. Write package.lua manifest
|
||||
packager.write_manifest(r, tarball_sha256, pkg_dir, opts.repo)
|
||||
|
||||
-- 11. Update index
|
||||
if not opts.no_index then
|
||||
indexer.generate(packages_dir)
|
||||
else
|
||||
log.info("--no-index: skipping index update")
|
||||
end
|
||||
|
||||
-- 12. Cleanup
|
||||
if not opts.keep_work then
|
||||
path.run("rm -rf " .. path.quote(work_dir))
|
||||
else
|
||||
log.info(("--keep-work: work directory preserved at %s"):format(work_dir))
|
||||
end
|
||||
|
||||
log.ok(("%s-%s built successfully"):format(r.name, r.version))
|
||||
log.info(("output: %s"):format(pkg_dir))
|
||||
end
|
||||
|
||||
local function run_index(opts)
|
||||
local packages_dir = path.join(opts.output, "packages")
|
||||
indexer.generate(packages_dir)
|
||||
end
|
||||
|
||||
---------------------------------------------------------------------------
|
||||
-- Main
|
||||
-- Dispatch
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
local args = {}
|
||||
@@ -270,9 +218,11 @@ if opts.help then
|
||||
end
|
||||
|
||||
if opts.index then
|
||||
run_index(opts)
|
||||
indexer.generate(path.join(opts.output, "packages"))
|
||||
elseif opts.all then
|
||||
build.all(opts.all, opts, arg[0])
|
||||
elseif opts.recipe then
|
||||
local ok, err = pcall(run_build, opts.recipe, opts)
|
||||
local ok, err = pcall(build.single, opts.recipe, opts)
|
||||
if not ok then
|
||||
log.fatal(tostring(err))
|
||||
end
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
-- build.lua -- single-recipe pipeline and batch dispatcher.
|
||||
--
|
||||
-- build_single(recipe_path, opts) → runs the full pipeline for one recipe
|
||||
-- build_all(dir, opts, script_path) → batch build (sequential or parallel)
|
||||
|
||||
local build = {}
|
||||
|
||||
local path = require("path")
|
||||
local log = require("log")
|
||||
local recipe = require("recipe")
|
||||
local builder = require("builder")
|
||||
local packager = require("packager")
|
||||
local indexer = require("indexer")
|
||||
|
||||
---------------------------------------------------------------------------
|
||||
-- Single recipe pipeline
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
function build.single(recipe_path, opts)
|
||||
local r, err = recipe.load(recipe_path)
|
||||
if not r then error(("recipe error: %s"):format(tostring(err)), 0) end
|
||||
|
||||
local packages_dir = path.join(opts.output, "packages")
|
||||
local pkg_dir = path.join(packages_dir, r.name)
|
||||
|
||||
if path.exists(pkg_dir) and not opts.force then
|
||||
error(("package %s already exists at %s (use --force to overwrite)"):format(
|
||||
r.name, pkg_dir), 0)
|
||||
end
|
||||
path.mkdir_p(pkg_dir)
|
||||
|
||||
local stamp = os.date("%Y%m%d-%H%M%S")
|
||||
local work_dir = path.join("/tmp/zeta-makepkg", r.name .. "-" .. stamp)
|
||||
local stage_dir = path.join(work_dir, "stage")
|
||||
path.mkdir_p(stage_dir)
|
||||
|
||||
log.info(("work directory: %s"):format(work_dir))
|
||||
|
||||
local cache_dir = path.join(os.getenv("HOME") or "/tmp", ".cache", "zeta-makepkg", "sources")
|
||||
local cache_file = builder.fetch_source(r, cache_dir)
|
||||
local source_dir = builder.extract_source(r, cache_file, work_dir)
|
||||
|
||||
builder.run_build(r, source_dir, stage_dir, { jobs = opts.jobs })
|
||||
|
||||
if r.test then
|
||||
builder.run_test(r.test, stage_dir)
|
||||
else
|
||||
log.info("no test command — skipping verification")
|
||||
end
|
||||
|
||||
local tarball_path = path.join(pkg_dir, r.name .. "-" .. r.version .. ".tar.gz")
|
||||
local sha256 = packager.create_tarball(stage_dir, tarball_path)
|
||||
packager.write_manifest(r, sha256, pkg_dir, opts.repo)
|
||||
|
||||
if not opts.no_index then
|
||||
indexer.generate(packages_dir)
|
||||
else
|
||||
log.info("--no-index: skipping index update")
|
||||
end
|
||||
|
||||
if not opts.keep_work then
|
||||
path.run("rm -rf " .. path.quote(work_dir))
|
||||
else
|
||||
log.info(("--keep-work: work directory preserved at %s"):format(work_dir))
|
||||
end
|
||||
|
||||
log.ok(("%s-%s built successfully"):format(r.name, r.version))
|
||||
log.info(("output: %s"):format(pkg_dir))
|
||||
end
|
||||
|
||||
---------------------------------------------------------------------------
|
||||
-- Collect recipes from a directory
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
local function collect_recipes(dir)
|
||||
local recipes = {}
|
||||
local f = io.popen("find " .. path.quote(dir)
|
||||
.. " -maxdepth 1 -name '*.recipe' -type f 2>/dev/null | sort")
|
||||
if f then
|
||||
for line in f:lines() do
|
||||
if line ~= "" then recipes[#recipes + 1] = line end
|
||||
end
|
||||
f:close()
|
||||
end
|
||||
return recipes
|
||||
end
|
||||
|
||||
---------------------------------------------------------------------------
|
||||
-- Batch build
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
function build.all(dir, opts, script_path)
|
||||
if not path.exists(dir) then
|
||||
error(("directory not found: %s"):format(dir), 0)
|
||||
end
|
||||
|
||||
log.step(("scanning %s for recipes"):format(dir))
|
||||
|
||||
local recipes = collect_recipes(dir)
|
||||
if #recipes == 0 then
|
||||
error(("no .recipe files found in %s"):format(dir), 0)
|
||||
end
|
||||
|
||||
log.info(("found %d recipe(s)"):format(#recipes))
|
||||
|
||||
local batch_opts = {
|
||||
output = opts.output,
|
||||
no_index = true,
|
||||
keep_work = opts.keep_work,
|
||||
force = opts.force,
|
||||
jobs = opts.jobs,
|
||||
repo = opts.repo,
|
||||
}
|
||||
|
||||
local failed = {}
|
||||
local workers = opts.workers or 1
|
||||
|
||||
if workers <= 1 then
|
||||
for _, recipe_path in ipairs(recipes) do
|
||||
io.write("\n" .. string.rep("─", 48) .. "\n")
|
||||
log.step(("building %s"):format(path.basename(recipe_path)))
|
||||
local ok, err = pcall(build.single, recipe_path, batch_opts)
|
||||
if not ok then
|
||||
log.error(tostring(err))
|
||||
failed[#failed + 1] = { path = recipe_path, err = tostring(err) }
|
||||
end
|
||||
end
|
||||
else
|
||||
local task_dir = "/tmp/zeta-makepkg-workers"
|
||||
path.run("rm -rf " .. path.quote(task_dir))
|
||||
path.mkdir_p(task_dir)
|
||||
|
||||
local jobs_flag = opts.jobs and (" -j" .. opts.jobs) or ""
|
||||
local running = 0
|
||||
|
||||
local function count_alive()
|
||||
local n = 0
|
||||
for i = 1, #recipes do
|
||||
if not path.exists(task_dir .. "/" .. i .. ".done") then n = n + 1 end
|
||||
end
|
||||
return n
|
||||
end
|
||||
|
||||
for i, recipe_path in ipairs(recipes) do
|
||||
while running >= workers do
|
||||
os.execute("sleep 0.5")
|
||||
running = count_alive()
|
||||
end
|
||||
|
||||
local done_file = task_dir .. "/" .. i .. ".done"
|
||||
local log_file = task_dir .. "/" .. i .. ".log"
|
||||
local cmd = string.format(
|
||||
"(%s %s %s --output %s --no-index --force --repo %s > %s 2>&1; echo $? > %s) &",
|
||||
path.quote(script_path), jobs_flag, path.quote(recipe_path),
|
||||
path.quote(opts.output), path.quote(opts.repo),
|
||||
path.quote(log_file), path.quote(done_file))
|
||||
|
||||
os.execute(cmd)
|
||||
running = running + 1
|
||||
log.info(("[%d/%d] spawned %s"):format(i, #recipes, path.basename(recipe_path)))
|
||||
end
|
||||
|
||||
log.step("waiting for workers...")
|
||||
while count_alive() > 0 do
|
||||
os.execute("sleep 0.5")
|
||||
end
|
||||
|
||||
for i = 1, #recipes do
|
||||
local done_file = task_dir .. "/" .. i .. ".done"
|
||||
local f = io.open(done_file, "r")
|
||||
local exit_code = f and tonumber(f:read("*l")) or 1
|
||||
if f then f:close() end
|
||||
if exit_code ~= 0 then
|
||||
local err_msg = ("exit code %d"):format(exit_code)
|
||||
local lf = io.open(task_dir .. "/" .. i .. ".log", "r")
|
||||
if lf then
|
||||
for line in lf:lines() do
|
||||
if line:match("error") then err_msg = line end
|
||||
end
|
||||
lf:close()
|
||||
end
|
||||
failed[#failed + 1] = { path = recipes[i], err = err_msg }
|
||||
else
|
||||
log.ok(path.basename(recipes[i]))
|
||||
end
|
||||
end
|
||||
|
||||
path.run("rm -rf " .. path.quote(task_dir))
|
||||
end
|
||||
|
||||
io.write("\n" .. string.rep("─", 48) .. "\n")
|
||||
if not opts.no_index then
|
||||
indexer.generate(path.join(opts.output, "packages"))
|
||||
end
|
||||
|
||||
io.write("\n")
|
||||
local built = #recipes - #failed
|
||||
log.ok(("%d built, %d failed"):format(built, #failed))
|
||||
if #failed > 0 then
|
||||
log.warn("failed recipes:")
|
||||
for _, f in ipairs(failed) do
|
||||
log.warn((" %s — %s"):format(path.basename(f.path), f.err))
|
||||
end
|
||||
os.exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
return build
|
||||
Reference in New Issue
Block a user