Files
antelope/source/antelope/build/pool.d
T
2026-07-28 22:17:49 -04:00

923 lines
32 KiB
D

/// Parallel build worker pool with dependency-aware scheduling.
///
/// Uses D's `std.concurrency` Actor model for worker coordination:
/// - Workers are spawned as OS threads via `spawn()`
/// - The main thread acts as coordinator: maintains the ready queue,
/// dispatches BuildJob messages to workers, and receives JobDone results
/// - No shared mutable state — all communication is via message passing
///
/// Features:
/// - Ready queue with critical-path priority sorting (load-aware scheduling)
/// - Fine-grained dispatch: targets become ready immediately when their
/// last prerequisite completes (not batched)
/// - Worker pool reuse: N threads spawned once, reused for all targets
/// - Configurable job limits (-jN)
/// - Correct failure propagation: failed targets mark dependents as skipped
/// - .NOTPARALLEL support: targets run exclusively when all workers idle
/// - Output buffering via OutputManager for atomic per-target printing
module antelope.build.pool;
import core.thread : Thread;
import std.concurrency : spawn, send, receive, receiveOnly, receiveTimeout,
Tid, thisTid, ownerTid;
import std.algorithm : sort;
import std.conv : to;
import antelope.build.target;
import antelope.build.graph;
import antelope.build.dependency;
import antelope.build.output;
import antelope.build.executor;
import antelope.shell.process;
import antelope.shell.environment;
import antelope.filesystem.timestamps;
import antelope.compatibility.parallel;
import antelope.compatibility.vpath;
import antelope.diagnostics.output;
// ── Messages ────────────────────────────────────────────────────────────
/// Sent from coordinator to worker: "build this target."
struct BuildJob
{
string targetName; /// Target to build
immutable(string)[] expandedRecipe; /// Pre-expanded recipe lines
immutable(bool)[] ignoreErrors; /// Per-line: - prefix
immutable(bool)[] silent; /// Per-line: @ prefix
immutable(string)[] execEnv; /// KEY=VALUE environment
bool dryRun; /// If true, echo but don't execute
}
/// Sent from worker to coordinator: "target built (or failed)."
struct JobDone
{
Tid workerTid; /// Which worker completed
string targetName; /// Which target was built
bool success; /// True if all recipe lines succeeded
int exitCode; /// Last non-zero exit code (0 on success)
bool hadEcho; /// True if any recipe line was echoed (non-@)
immutable(string)[] stdoutLines; /// Captured stdout lines
immutable(string)[] stderrLines; /// Captured stderr lines
}
/// Sent from coordinator to worker: "exit your loop."
struct Shutdown {}
// ── WorkerPool ───────────────────────────────────────────────────────────
/// Manages parallel build execution.
///
/// Usage:
/// ```d
/// auto pool = WorkerPool(config.jobs);
/// int exitCode = pool.build(graph, rootTargets, config, env, expander, output, vpath);
/// ```
struct WorkerPool
{
private:
uint numWorkers; /// Number of worker threads (= -j value)
Tid[] workerTids; /// Tids of spawned workers
bool started; /// Whether workers have been spawned
// Build state (populated during build())
DependencyGraph* graph;
ParallelConfig* parallelConfig;
OutputManager* outputMgr;
VPathConfig* vpathConfig; /// VPATH for needsRebuild in dequeueDependents
bool[string] notParallelSet; /// Targets marked .NOTPARALLEL
bool[string] failedSet; /// Targets that failed (for propagation)
bool[string] skippedSet; /// Targets blocked by failed prereqs
// Ready queue (sorted by descending criticalWeight)
// Stored as indices into graph.targets
size_t[] readyQueue;
// Dependency tracking
size_t[string] remainingDeps;
// .JOBS throttle: tracks active workers per job-limit value.
size_t[size_t] activeByLimit;
// Completed target counter (shared between build() and dequeueDependents).
size_t completedCount;
public:
/// Create a worker pool with the given number of workers.
///
/// If `numWorkers` is 0, it defaults to the number of CPU cores.
/// If `numWorkers` is 1, the build runs serially without spawning threads.
static WorkerPool create(uint numWorkers = 0)
{
import std.parallelism : totalCPUs;
WorkerPool pool;
if (numWorkers == 0)
pool.numWorkers = totalCPUs;
else
pool.numWorkers = numWorkers;
return pool;
}
/// Jobserver file descriptors for cross-process token coordination.
/// readFd is passed to child processes via MAKEFLAGS; writeFd is
/// held by the parent to return tokens after job completion.
struct JobserverPipe
{
int readFd = -1; /// Read end — child processes consume tokens here
int writeFd = -1; /// Write end — parent writes tokens back on completion
bool active; /// Whether the jobserver is operational
}
/// Create a jobserver pipe with `nTokens` initial tokens.
///
/// Writes N bytes to the pipe so that up to N jobs can run
/// concurrently across recursive $(MAKE) invocations. Each job
/// reads one byte before starting; the byte is written back on
/// completion.
static JobserverPipe createJobserverPipe(uint nTokens)
{
version (Posix)
{
import core.sys.posix.unistd : pipe, read, write, close;
import core.sys.posix.fcntl : fcntl, F_SETFD, FD_CLOEXEC;
JobserverPipe js;
int[2] fds;
if (pipe(fds) != 0)
return js;
// Write end: mark close-on-exec so child processes only
// inherit the read end.
fcntl(fds[1], F_SETFD, FD_CLOEXEC);
js.readFd = fds[0];
js.writeFd = fds[1];
js.active = true;
// Seed the pipe with N tokens (one byte each).
ubyte token = 0;
for (uint i = 0; i < nTokens; i++)
write(js.writeFd, &token, 1);
return js;
}
else
{
// Non-POSIX: jobserver not supported.
return JobserverPipe();
}
}
/// Consume one token from the jobserver pipe (blocking).
/// Returns true if a token was acquired, false on error.
private static bool acquireJobserverToken(int readFd)
{
version (Posix)
{
import core.sys.posix.unistd : read;
ubyte token;
return read(readFd, &token, 1) == 1;
}
else
return false;
}
/// Return one token to the jobserver pipe.
private static bool releaseJobserverToken(int writeFd)
{
version (Posix)
{
import core.sys.posix.unistd : write;
ubyte token = 0;
return write(writeFd, &token, 1) == 1;
}
else
return false;
}
/// Run the build for the given root targets.
///
/// Params:
/// graph = Dependency graph with all targets (mutated: runtime state fields)
/// roots = Root target names to build (e.g., ["all"])
/// config = Parallel config (jobs, notParallelTargets, output sync mode)
/// env = Build environment (passed to recipe subprocesses)
/// expander = Variable expansion delegate
/// output = Output buffer manager
/// vpath = VPATH config (for needsRebuild checks)
/// baseExecEnv = KEY=VALUE pairs added to every job's env (e.g., SHELL, MAKEFLAGS)
/// dryRun = Print commands without executing
/// silent = Suppress command echoing
///
/// Returns: exit code (0 = success, non-zero = failure).
int build(
ref DependencyGraph graph,
string[] roots,
ref ParallelConfig config,
Environment* env,
string delegate(string, string, string[], string) expander,
OutputManager* output,
VPathConfig* vpath,
string[] baseExecEnv = [],
bool dryRun = false,
bool silent = false)
{
import antelope.filesystem.timestamps;
this.graph = &graph;
this.parallelConfig = &config;
this.outputMgr = output;
this.vpathConfig = vpath;
// Determine actual worker count.
uint nWorkers = config.jobs;
if (nWorkers == 0)
{
import std.parallelism : totalCPUs;
nWorkers = totalCPUs;
}
// Reset per-build state.
readyQueue = [];
failedSet = null;
skippedSet = null;
remainingDeps = null;
// Serial mode shortcut.
if (nWorkers <= 1)
return buildSerial(graph, roots, config, env, expander, output,
vpath, baseExecEnv, dryRun, silent);
// Build the combined transitive closure for all root targets.
bool[string] inClosure;
foreach (root; roots)
{
auto closure = graph.transitiveClosure(root);
foreach (name; closure)
inClosure[name] = true;
}
if (inClosure.length == 0)
return 0;
// Set up scheduling state.
graph.resetSchedulingState();
graph.buildReverseEdges();
import antelope.build.dependency;
foreach (root; roots)
computeCriticalWeights(graph, root);
graph.computeRemainingDeps();
// Copy remainingDeps for fast lookup BEFORE the init loop so
// that dequeueDependents (called for up-to-date targets during
// initial ready-queue construction) can read from it.
remainingDeps = null;
foreach (ref t; graph.targets)
if (t.name in inClosure)
remainingDeps[t.name] = t.remainingDeps;
// Populate initial ready queue: targets with remainingDeps == 0
// that actually need building and are in the closure.
size_t[] initialReady;
foreach (i, ref t; graph.targets)
{
if (t.name !in inClosure)
continue;
if (t.remainingDeps != 0)
continue;
if (!needsRebuild(t.name, t.prerequisites,
&graph.phonyTargets, vpath, &t.orderOnlyPrereqs))
{
t.state = BuildState.completed;
// Notify dependents so they can become ready.
dequeueDependents(t.name);
continue;
}
initialReady ~= i;
}
// Sort initial-ready targets by descending critical weight.
initialReady.sort!((a, b) =>
graph.targets[a].criticalWeight > graph.targets[b].criticalWeight);
// Merge initial-ready targets with any targets that became ready
// during the init loop (via dequeueDependents for up-to-date
// prerequisites). Initial-ready goes first (already filtered
// by needsRebuild), then dequeueDependents-added targets.
readyQueue = initialReady ~ readyQueue;
size_t totalTargets = 0;
foreach (i, ref t; graph.targets)
if (t.name in inClosure && t.state != BuildState.completed)
totalTargets++;
// Spawn worker threads.
workerTids.length = 0;
for (uint i = 0; i < nWorkers; i++)
{
auto tid = spawn(&workerFunc);
workerTids ~= tid;
}
this.numWorkers = nWorkers;
this.started = true;
// Seed idle worker queue: all workers start idle.
Tid[] idleWorkers = workerTids.dup;
// Send initial batch of jobs (respect .JOBS and .NOTPARALLEL limits).
completedCount = 0;
while (readyQueue.length > 0 && idleWorkers.length > 0)
{
size_t idx = readyQueue[0];
// NOTPARALLEL: only dispatch when all other workers are idle.
if (idx < graph.targets.length &&
graph.targets[idx].name in notParallelSet &&
idleWorkers.length != workerTids.length)
break;
// .JOBS limit: throttle if this target has a job limit.
if (idx < graph.targets.length &&
graph.targets[idx].jobLimit > 0)
{
size_t busy = workerTids.length - idleWorkers.length;
if (busy >= graph.targets[idx].jobLimit)
break;
}
readyQueue = readyQueue[1 .. $];
auto job = makeBuildJob(idx, env, expander, baseExecEnv, dryRun);
if (job.expandedRecipe.length > 0)
{
graph.targets[idx].state = BuildState.running;
send(idleWorkers[$ - 1], job);
idleWorkers = idleWorkers[0 .. $ - 1];
if (graph.targets[idx].jobLimit > 0)
activeByLimit[graph.targets[idx].jobLimit]++;
}
else
{
// Target with no recipe: mark complete immediately.
graph.targets[idx].state = BuildState.completed;
dequeueDependents(graph.targets[idx].name);
completedCount++;
}
}
// Coordinator loop: dispatch → receive → process → repeat.
bool hasFailure = false;
while (completedCount < totalTargets)
{
// Phase 1: Dispatch as many ready targets as possible
// to idle workers.
while (readyQueue.length > 0 && idleWorkers.length > 0)
{
size_t idx = readyQueue[0];
// NOTPARALLEL: only dispatch when all other workers are idle.
if (idx < graph.targets.length &&
graph.targets[idx].name in notParallelSet &&
idleWorkers.length != workerTids.length)
break;
// .JOBS limit: throttle if this target has a job limit.
// Tracks only workers running .JOBS-limited targets of
// the same limit value, not total busy workers.
if (idx < graph.targets.length &&
graph.targets[idx].jobLimit > 0)
{
size_t limit = graph.targets[idx].jobLimit;
auto countPtr = limit in activeByLimit;
size_t active = countPtr ? *countPtr : 0;
if (active >= limit)
break;
}
readyQueue = readyQueue[1 .. $];
auto job = makeBuildJob(idx, env, expander, baseExecEnv, dryRun);
if (job.expandedRecipe.length > 0)
{
graph.targets[idx].state = BuildState.running;
send(idleWorkers[$ - 1], job);
idleWorkers = idleWorkers[0 .. $ - 1];
if (graph.targets[idx].jobLimit > 0)
activeByLimit[graph.targets[idx].jobLimit]++;
}
else
{
// Target with no recipe: complete immediately.
graph.targets[idx].state = BuildState.completed;
dequeueDependents(graph.targets[idx].name);
completedCount++;
}
}
// Phase 2: Check termination.
if (idleWorkers.length == workerTids.length)
{
// All workers idle. If work remains, targets are
// blocked (waiting for failed/skipped prereqs).
if (completedCount < totalTargets)
{
foreach (ref t; graph.targets)
{
if (t.name !in inClosure)
continue;
if (t.state == BuildState.pending)
{
t.state = BuildState.skipped;
skippedSet[t.name] = true;
completedCount++;
if (output && output.buffered)
output.flush(t.name);
}
}
}
break;
}
// Phase 3: Wait for a worker result.
auto done = receiveOnly!JobDone();
// Worker is now idle.
idleWorkers ~= done.workerTid;
// Buffer output into OutputManager.
if (output)
{
foreach (line; done.stdoutLines)
output.bufferStdout(done.targetName, cast(string) line);
foreach (line; done.stderrLines)
output.bufferStderr(done.targetName, cast(string) line);
if (done.hadEcho)
output.markEchoed(done.targetName);
}
// Phase 4: Process result.
auto tp = graph.findTarget(done.targetName);
if (tp !is null)
{
// Decrement .JOBS throttle counter if this target had a limit.
if (tp.jobLimit > 0)
{
auto countPtr = tp.jobLimit in activeByLimit;
if (countPtr && *countPtr > 0)
(*countPtr)--;
}
if (done.success)
{
tp.state = BuildState.completed;
log(LogLevel.dbg, "[" ~ done.targetName ~ "] completed");
dequeueDependents(done.targetName);
}
else
{
tp.state = BuildState.failed;
failedSet[done.targetName] = true;
hasFailure = true;
log(LogLevel.dbg, "[" ~ done.targetName ~
"] FAILED (exit " ~ done.exitCode.to!string ~ ")");
propagateFailure(done.targetName, inClosure);
}
}
// Flush buffered output.
if (output && output.buffered)
output.flush(done.targetName);
completedCount++;
// Loop back to Phase 1 (dispatch newly-ready targets).
}
// Shutdown all workers.
foreach (tid; workerTids)
{
try { send(tid, Shutdown()); } catch (Exception) {}
}
// Collect any late messages (workers may have sent results
// that haven't been received yet due to timing).
// Use a short timeout to drain the queue.
import core.time : dur;
while (true)
{
auto msg = receiveTimeout(dur!"msecs"(100), (JobDone d) => true);
if (!msg)
break;
}
workerTids = [];
started = false;
return hasFailure ? 1 : 0;
}
private:
/// Serial fallback for -j1 or single-worker builds.
int buildSerial(
ref DependencyGraph graph,
string[] roots,
ref ParallelConfig config,
Environment* env,
string delegate(string, string, string[], string) expander,
OutputManager* output,
VPathConfig* vpath,
string[] baseExecEnv,
bool dryRun,
bool silent)
{
import antelope.filesystem.timestamps : needsRebuild;
bool hasFailure;
foreach (root; roots)
{
auto batches = resolveDependencies(graph, root);
foreach (batch; batches)
{
foreach (ref t; batch)
{
if (!needsRebuild(t.name, t.prerequisites,
&graph.phonyTargets, vpath, &t.orderOnlyPrereqs))
continue;
string[] execEnv = baseExecEnv.dup;
auto result = executeTarget(t, execEnv, expander,
output, dryRun, silent);
if (output && output.buffered)
output.flush(t.name);
if (!result.success)
{
hasFailure = true;
goto done;
}
}
}
}
done:
return hasFailure ? 1 : 0;
}
/// Worker thread function.
static void workerFunc()
{
import std.string : stripLeft;
bool running = true;
while (running)
{
receive(
(BuildJob job) {
JobDone done;
done.workerTid = thisTid;
done.targetName = job.targetName;
done.success = true;
done.exitCode = 0;
done.hadEcho = false;
// Build mutable output buffers, freeze before sending.
string[] outLines;
string[] errLines;
if (job.expandedRecipe.length == 0)
{
done.stdoutLines = outLines.idup;
done.stderrLines = errLines.idup;
send(ownerTid, done);
return;
}
foreach (i, line; job.expandedRecipe)
{
// line is immutable(string); cast to string for stdlib.
string sline = cast(string) line;
if (sline.stripLeft.length == 0)
continue;
bool ignoreErrors = i < job.ignoreErrors.length
? cast(bool) job.ignoreErrors[i] : false;
bool silent = i < job.silent.length
? cast(bool) job.silent[i] : false;
// Echo
if (!silent)
{
done.hadEcho = true;
outLines ~= sline;
}
// Dry run: skip actual execution.
if (job.dryRun)
continue;
// Execute. Cast execEnv back to mutable for runProcessPiped.
auto ph = runProcessPiped(sline, cast(string[]) job.execEnv);
// Read pipes.
try
{
import std.string : chomp;
foreach (pl; ph.stdoutPipe.byLine)
{
string s = pl.chomp().idup;
outLines ~= s;
}
foreach (pl; ph.stderrPipe.byLine)
{
string s = pl.chomp().idup;
errLines ~= s;
}
}
catch (Exception e)
{
// Log pipe read errors but don't abort the build.
log(LogLevel.dbg, "[" ~ job.targetName ~
"] pipe read error: " ~ e.msg);
}
int code = ph.waitFor();
if (code != 0 && !ignoreErrors)
{
ph.closePipes();
done.success = false;
done.exitCode = code;
done.stdoutLines = outLines.idup;
done.stderrLines = errLines.idup;
send(ownerTid, done);
return;
}
ph.closePipes();
}
done.stdoutLines = outLines.idup;
done.stderrLines = errLines.idup;
send(ownerTid, done);
},
(Shutdown _) {
running = false;
}
);
}
}
/// Create a BuildJob for the target at graph index `idx`.
BuildJob makeBuildJob(
size_t idx,
Environment* env,
string delegate(string, string, string[], string) expander,
string[] baseExecEnv,
bool dryRun)
{
auto t = &graph.targets[idx];
// Build mutable arrays, then freeze to immutable for sending.
string[] recipeLines;
bool[] ignoreErrs;
bool[] silents;
foreach (line; t.recipe)
{
// Expand variables in the recipe.
string expanded = expander(line, t.name,
t.prerequisites, t.stem);
// Strip and classify prefix characters.
import std.string : stripLeft;
string trimmed = expanded.stripLeft();
bool ignoreErrors;
bool silent;
if (trimmed.length > 0)
{
bool stripping = true;
while (stripping && trimmed.length > 0)
{
stripping = false;
switch (trimmed[0])
{
case '@':
silent = true;
trimmed = trimmed[1 .. $];
stripping = true;
break;
case '-':
ignoreErrors = true;
trimmed = trimmed[1 .. $];
stripping = true;
break;
case '+':
trimmed = trimmed[1 .. $];
stripping = true;
break;
default:
break;
}
}
}
if (trimmed.length == 0)
continue;
recipeLines ~= trimmed;
ignoreErrs ~= ignoreErrors;
silents ~= silent;
}
// Build execEnv.
string[] execEnvArr = baseExecEnv.dup;
// Freeze arrays to immutable for std.concurrency message passing.
BuildJob job;
job.targetName = t.name;
job.expandedRecipe = recipeLines.idup;
job.ignoreErrors = ignoreErrs.idup;
job.silent = silents.idup;
job.execEnv = execEnvArr.idup;
job.dryRun = dryRun;
return job;
}
/// Decrement remainingDeps for all dependents of `completedTarget`.
/// Any dependent that reaches 0 remaining deps is added to the
/// ready queue (sorted by descending critical weight).
void dequeueDependents(string completedTarget)
{
auto tp = graph.findTarget(completedTarget);
if (tp is null)
return;
foreach (depName; tp.dependents)
{
// Skip if not in our tracking (could be outside closure).
auto depPtr = depName in remainingDeps;
if (depPtr is null)
continue;
if (*depPtr == 0)
continue; // Already ready or completed
(*depPtr)--;
if (*depPtr == 0 && !(depName in skippedSet))
{
// Target is now ready — check if it needs building.
auto dep = graph.findTarget(depName);
if (dep is null || dep.state != BuildState.pending)
continue;
// Check up-to-date: targets that become ready via
// dequeueDependents were NOT filtered during the init
// loop (which only checks initially-zero-dep targets).
import antelope.filesystem.timestamps : needsRebuild;
if (!needsRebuild(dep.name, dep.prerequisites,
&graph.phonyTargets, vpathConfig, &dep.orderOnlyPrereqs))
{
dep.state = BuildState.completed;
completedCount++;
dequeueDependents(dep.name);
continue;
}
// Insert sorted by descending critical weight.
bool inserted;
foreach (i, qi; readyQueue)
{
if (dep.criticalWeight > graph.targets[qi].criticalWeight)
{
readyQueue = readyQueue[0 .. i] ~
[cast(size_t)(dep - graph.targets.ptr)] ~
readyQueue[i .. $];
inserted = true;
break;
}
}
if (!inserted)
readyQueue ~= cast(size_t)(dep - graph.targets.ptr);
}
}
}
/// Mark all dependents of a failed target as skipped.
/// Recursively propagates: if A depends on B and B fails,
/// A is skipped; if C depends on A, C is also skipped.
void propagateFailure(string failedTarget, ref bool[string] inClosure)
{
import std.algorithm : canFind;
string[] stack = [failedTarget];
while (stack.length > 0)
{
string current = stack[$ - 1];
stack = stack[0 .. $ - 1];
auto tp = graph.findTarget(current);
if (tp is null)
continue;
foreach (depName; tp.dependents)
{
if (depName in skippedSet || depName in failedSet)
continue;
if (depName !in inClosure)
continue;
auto dep = graph.findTarget(depName);
if (dep is null)
continue;
dep.state = BuildState.skipped;
skippedSet[depName] = true;
stack ~= depName;
}
}
}
}
// ── Unittests ────────────────────────────────────────────────────────────
///
unittest
{
// Build a trivial graph with one target (no recipe).
DependencyGraph g;
g.addTarget(Target("leaf", TargetKind.file, [], []));
g.buildReverseEdges();
g.computeRemainingDeps();
computeCriticalWeights(g, "leaf");
ParallelConfig pc;
pc.jobs = 2;
auto om = new OutputManager();
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto pool = WorkerPool.create(2);
int code = pool.build(g, ["leaf"], pc, null, &expand, &om, null);
assert(code == 0);
assert(g.findTarget("leaf").state == BuildState.completed);
}
///
unittest
{
// Chain: a → b → c (all no recipe, always succeed).
DependencyGraph g;
g.addTarget(Target("c", TargetKind.file, [], []));
g.addTarget(Target("b", TargetKind.file, ["c"], []));
g.addTarget(Target("a", TargetKind.file, ["b"], []));
// Ensure all are in the closure.
g.buildReverseEdges();
g.computeRemainingDeps();
computeCriticalWeights(g, "a");
ParallelConfig pc;
pc.jobs = 2;
auto om = new OutputManager();
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto pool = WorkerPool.create(2);
int code = pool.build(g, ["a"], pc, null, &expand, &om, null);
assert(code == 0);
assert(g.findTarget("a").state == BuildState.completed);
assert(g.findTarget("b").state == BuildState.completed);
assert(g.findTarget("c").state == BuildState.completed);
}
// Regression: serial mode (jobs=1) should work.
unittest
{
DependencyGraph g;
g.addTarget(Target("x", TargetKind.file, [], []));
g.buildReverseEdges();
g.computeRemainingDeps();
computeCriticalWeights(g, "x");
ParallelConfig pc;
pc.jobs = 1;
auto om = new OutputManager();
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto pool = WorkerPool.create(1);
int code = pool.build(g, ["x"], pc, null, &expand, &om, null);
assert(code == 0);
}