huntedbytheirs/antelope
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

Concurrency

Browse Source
This commit is contained in:
huntedbytheirs
2026-07-28 22:17:49 -04:00
parent f235a0cf78
commit 969e36c5ff
29 changed files with 2966 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/antelope/build/dependency.d
+168 -3
View File
@@ -1,9 +1,8 @@
/// Dependency resolution and ordering logic.
///
/// Uses Kahn's algorithm (BFS-based topological sort) to produce ordered
/// build batches. Each batch contains targets that can be built in parallel.
/// The first batch contains leaf targets (no unresolved prereqs in the
/// graph), and the last batch contains the requested root target.
/// build batches, plus critical-path weight computation for load-aware
/// scheduling.
module antelope.build.dependency;
import antelope.build.graph;
@@ -152,6 +151,124 @@ Target[][] resolveDependencies(DependencyGraph graph, string target)
return batches;
}
/// Compute critical-path weights for every target reachable from `root`.
///
/// The critical-path weight of a target is:
/// weight = recipe.length + max(weight of each successor)
///
/// A "successor" is any target that depends on this target (i.e., a
/// target listing this one as a prerequisite). This is the reverse
/// of the usual dependency direction — we compute from the root
/// backward to the leaves.
///
/// Leaf nodes (targets with no in-graph dependents) have weight = recipe.length.
/// Root nodes accumulate the full chain of work beneath them.
///
/// The resulting weights are written directly into each `Target.criticalWeight`
/// field. The caller sorts the ready queue by descending `criticalWeight`
/// to prioritize targets on the critical path.
///
/// Params:
/// graph = The dependency graph with reverse edges already populated
/// via `DependencyGraph.buildReverseEdges()`.
/// root = The root target name to start the weight computation from.
void computeCriticalWeights(ref DependencyGraph graph, string root)
{
import std.algorithm : max;
// Only consider targets reachable from the root.
bool[string] reachable;
{
string[] stack = [root];
while (stack.length > 0)
{
string current = stack[$ - 1];
stack = stack[0 .. $ - 1];
if (current in reachable)
continue;
reachable[current] = true;
auto tp = graph.findTarget(current);
if (tp is null)
continue;
foreach (dep; tp.prerequisites ~ tp.orderOnlyPrereqs)
if (graph.hasTarget(dep) && dep !in reachable)
stack ~= dep;
}
}
// Build a dependency map: node → all in-graph prereqs
string[][string] prereqMap;
foreach (ref t; graph.targets)
{
if (t.name !in reachable)
continue;
foreach (p; t.prerequisites ~ t.orderOnlyPrereqs)
if (p in reachable && graph.hasTarget(p))
prereqMap[t.name] ~= p;
}
// Kahn-style topological order from leaves (in-degree 0) to root.
size_t[string] inDegree;
string[][string] dependentsMap; // prereq → dependents
foreach (name; reachable.keys)
inDegree[name] = 0;
foreach (name, prereqs; prereqMap)
{
inDegree[name] = prereqs.length;
foreach (p; prereqs)
dependentsMap[p] ~= name;
}
// Process in topological order: all of a node's prereqs are
// processed before the node itself, so their weights are finalised.
string[] queue;
foreach (name; reachable.keys)
if (inDegree[name] == 0)
queue ~= name;
string[] order;
while (queue.length > 0)
{
string current = queue[$ - 1];
queue = queue[0 .. $ - 1];
order ~= current;
auto deps = current in dependentsMap;
if (deps is null)
continue;
foreach (dep; *deps)
{
inDegree[dep]--;
if (inDegree[dep] == 0)
queue ~= dep;
}
}
// Now compute weights in topological order.
// order[0] = leaf, order[$-1] = root.
foreach (name; order)
{
auto tp = graph.findTarget(name);
if (tp is null)
continue;
size_t maxPrereqWeight = 0;
auto prereqs = name in prereqMap;
if (prereqs)
{
foreach (p; *prereqs)
{
auto pp = graph.findTarget(p);
if (pp !is null)
maxPrereqWeight = max(maxPrereqWeight, pp.criticalWeight);
}
}
tp.criticalWeight = tp.recipe.length + maxPrereqWeight;
}
}
///
unittest
{
@@ -211,3 +328,51 @@ unittest
assert(batches[1].length == 1);
assert(batches[1][0].name == "program");
}
/// Critical path weights: program(1) → main.o(1) → main.c(0) = 2
unittest
{
DependencyGraph g;
g.addTarget(Target("main.c", TargetKind.file, [], []));
g.addTarget(Target("main.o", TargetKind.file, ["main.c"],
["gcc -c main.c"]));
g.addTarget(Target("program", TargetKind.file, ["main.o"],
["gcc -o program main.o"]));
g.buildReverseEdges();
computeCriticalWeights(g, "program");
// main.c: no recipe, no prereqs → weight 0
auto mc = g.findTarget("main.c");
assert(mc !is null);
assert(mc.criticalWeight == 0);
// main.o: 1 recipe line, prereq main.c (weight 0) → weight 1
auto mo = g.findTarget("main.o");
assert(mo !is null);
assert(mo.criticalWeight == 1);
// program: 1 recipe line, prereq main.o (weight 1) → weight 2
auto prog = g.findTarget("program");
assert(prog !is null);
assert(prog.criticalWeight == 2);
}
/// Diamond dependency: root → a, b → leaf. Weights should reflect
/// that both branches are equal.
unittest
{
DependencyGraph g;
g.addTarget(Target("leaf", TargetKind.file, [], ["touch leaf"])); // weight 1
g.addTarget(Target("a", TargetKind.file, ["leaf"], ["cp leaf a"])); // weight 2
g.addTarget(Target("b", TargetKind.file, ["leaf"], ["cp leaf b"])); // weight 2
g.addTarget(Target("root", TargetKind.file, ["a", "b"], ["cat a b"])); // weight 3
g.buildReverseEdges();
computeCriticalWeights(g, "root");
assert(g.findTarget("leaf").criticalWeight == 1);
assert(g.findTarget("a").criticalWeight == 2);
assert(g.findTarget("b").criticalWeight == 2);
assert(g.findTarget("root").criticalWeight == 3);
}
source/antelope/build/executor.d
+251 -2
View File
@@ -1,17 +1,35 @@
/// Command execution engine — runs recipe lines and reports results.
///
/// Now supports parallel execution: `executeTarget()` spawns processes
/// with piped stdout/stderr for output buffering, while `execute()`
/// remains available for simple synchronous use.
module antelope.build.executor;
import antelope.shell.process;
import antelope.build.target;
import antelope.build.output;
import antelope.diagnostics.output;
/// Result of executing a single recipe line.
struct ExecResult
{
bool success; /// True if the command succeeded or ignoreErrors was set
string output; /// (reserved for future output capture)
string output; /// Captured stdout + stderr from the process
int exitCode; /// Exit code from the process
}
/// Execute a command string and return the result.
/// Result of executing all recipe lines for a single target.
struct JobResult
{
string targetName; /// Name of the target that was built
bool success; /// True if all recipe lines succeeded
int exitCode; /// Last non-zero exit code (0 if all succeeded)
string[] stdoutLines; /// Captured stdout, one entry per recipe line
string[] stderrLines; /// Captured stderr, one entry per recipe line
bool hadEcho; /// True if any non-@ recipe line was executed
}
/// Execute a command string and return the result (synchronous, no capture).
///
/// Handles GNU Make recipe prefix characters (@, -, +), then passes the
/// remaining line directly to /bin/sh. Tokenization is deliberately
@@ -60,3 +78,234 @@ ExecResult execute(string command, string[] environment = [])
bool ok = (code == 0 || ignoreErrors);
return ExecResult(ok, "", code);
}
/// Execute all recipe lines for a single target, capturing output.
///
/// Each recipe line is expanded (via the caller-supplied expander),
/// stripped of prefix characters, printed according to echo rules,
/// and executed via a piped subprocess. Output is buffered into the
/// supplied `OutputManager`.
///
/// The `expander` delegate is called to perform variable expansion
/// (e.g., $(CC), $@, $<) at execution time. It receives:
/// - The raw recipe line text
/// - The target name (for $@ expansion)
/// - The target's prerequisites (for $<, $^ expansion)
/// - The target's stem (for $* expansion in pattern rules)
///
/// Params:
/// t = The target to build
/// execEnv = Environment variables (KEY=VALUE) for the subprocess
/// expander = Delegate for variable expansion
/// output = Output buffer manager (may be null for live mode)
/// isDryRun = If true, print commands but don't execute
/// silentMode = If true, suppress echo of non-@ lines
///
/// Returns: JobResult with success flag and captured output.
JobResult executeTarget(
Target t,
string[] execEnv,
string delegate(string, string, string[], string) expander,
OutputManager* output,
bool isDryRun = false,
bool silentMode = false)
{
import std.string : stripLeft;
JobResult result;
result.targetName = t.name;
result.success = true;
result.exitCode = 0;
// Short-circuit: targets with no recipe are always "successful"
// (they exist on disk or are phony/intermediate markers).
if (t.recipe.length == 0)
return result;
foreach (recipeLine; t.recipe)
{
// Expand variables in the recipe line.
// Automatic variables ($@, $<, $^, $*, etc.) are resolved
// against the current target context.
string expanded = expander(recipeLine, t.name,
t.prerequisites, t.stem);
// Strip prefix characters to determine echo/error behaviour.
string trimmed = expanded.stripLeft();
bool ignoreErrors;
bool silent = silentMode;
if (trimmed.length > 0)
{
bool stripping = true;
while (stripping && trimmed.length > 0)
{
stripping = false;
switch (trimmed[0])
{
case '@':
silent = true;
trimmed = trimmed[1 .. $];
stripping = true;
break;
case '-':
ignoreErrors = true;
trimmed = trimmed[1 .. $];
stripping = true;
break;
case '+':
trimmed = trimmed[1 .. $];
stripping = true;
break;
default:
break;
}
}
}
if (trimmed.length == 0)
continue;
// Echo: print the command unless suppressed.
// GNU Make prints the expanded form.
string echoLine = expanded.stripLeft();
bool shouldEcho = !silent && echoLine.length > 0;
if (shouldEcho)
{
if (output)
{
output.bufferStdout(t.name, echoLine);
output.markEchoed(t.name);
result.hadEcho = true;
}
else
{
log(LogLevel.normal, echoLine);
}
}
// Dry run: skip execution.
if (isDryRun)
{
result.stdoutLines ~= shouldEcho ? echoLine : "";
continue;
}
// Execute the command with piped output.
auto ph = runProcessPiped(trimmed, execEnv);
// Read stdout and stderr from pipes.
string lineStdout;
string lineStderr;
// Simple line-by-line reading from the pipes.
// NOTE: stdout is read first, then stderr. If the child process
// fills its stderr pipe buffer (>64KB on Linux) before stdout
// is fully drained, both sides deadlock. For typical compiler
// output this is unlikely; a future fix should drain both pipes
// concurrently via select/poll or lightweight threads.
try
{
import std.string : chomp;
// Read stdout — ProcessHandle.stdoutPipe is a File directly.
foreach (line; ph.stdoutPipe.byLine)
{
string s = line.chomp().idup;
lineStdout ~= s ~ "\n";
if (output)
output.bufferStdout(t.name, s);
else
log(LogLevel.normal, s);
}
// Read stderr
foreach (line; ph.stderrPipe.byLine)
{
string s = line.chomp().idup;
lineStderr ~= s ~ "\n";
if (output)
output.bufferStderr(t.name, s);
else
log(LogLevel.normal, s);
}
}
catch (Exception e)
{
// Log pipe errors but continue — the process exit code
// will determine success/failure.
log(LogLevel.dbg, "[" ~ t.name ~ "] pipe read error: " ~ e.msg);
}
// Wait for the process.
int code = ph.waitFor();
ph.closePipes();
// Store output
result.stdoutLines ~= lineStdout;
result.stderrLines ~= lineStderr;
// Check result
if (code != 0 && !ignoreErrors)
{
result.success = false;
result.exitCode = code;
return result;
}
if (code != 0)
result.exitCode = code;
}
return result;
}
///
unittest
{
// Simple synchronous execution (no expansion needed for this test).
auto r = execute("echo hello");
assert(r.success);
assert(r.exitCode == 0);
}
///
unittest
{
// Error-tolerant execution.
auto r = execute("-exit 1");
assert(r.success); // - prefix ignores errors
assert(r.exitCode == 1);
}
///
unittest
{
// Target execution with piped output.
Target t;
t.name = "test";
t.recipe = ["echo hello world"];
// Identity expander (no variable substitution).
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto result = executeTarget(t, [], &expand, null, false, false);
assert(result.success);
assert(result.exitCode == 0);
}
///
unittest
{
// Target with failed recipe line.
Target t;
t.name = "failing";
t.recipe = ["exit 3"];
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto result = executeTarget(t, [], &expand, null, false, false);
assert(!result.success);
assert(result.exitCode == 3);
}
source/antelope/build/graph.d
+172
View File
@@ -57,6 +57,12 @@ struct DependencyGraph
// .PHONY handling: mark its prerequisites as phony targets
graph.handlePhony();
// .WAIT handling: split prerequisite groups with barriers
graph.handleWait();
// .JOBS handling: per-target job limits (native mode)
graph.handleJobs();
// Cycle detection: DFS with three-color marking
graph.detectCycles();
@@ -120,6 +126,9 @@ private:
}
}
// ── Public scheduling / special-target API ──────────────────────────
public:
/// Find the ".PHONY" target (if it exists) and mark all of its
/// prerequisites as phony targets.
void handlePhony()
@@ -137,6 +146,169 @@ private:
}
}
/// Process .WAIT special target: split prerequisite groups with barriers.
///
/// Syntax: `target: group1 .WAIT group2`
/// → group1 completes first, then group2 starts.
/// Adds implicit dependencies: each group2 target depends on each group1 target.
void handleWait()
{
foreach (ref t; targets)
{
ptrdiff_t waitPos = -1;
foreach (i, p; t.prerequisites)
{
if (p == ".WAIT")
{
waitPos = cast(ptrdiff_t) i;
break;
}
}
if (waitPos < 0)
continue;
string[] group1 = t.prerequisites[0 .. cast(size_t) waitPos];
string[] group2 = t.prerequisites[cast(size_t) waitPos + 1 .. $];
// Remove .WAIT from prerequisites.
t.prerequisites = group1 ~ group2;
// Add implicit deps: each group2 target depends on group1 targets.
foreach (g2name; group2)
{
auto g2 = findTarget(g2name);
if (g2 is null)
continue;
foreach (g1name; group1)
{
bool alreadyDepends;
foreach (p; g2.prerequisites)
if (p == g1name) { alreadyDepends = true; break; }
if (!alreadyDepends && g1name != g2.name)
g2.prerequisites ~= g1name;
}
}
}
}
/// Process .JOBS special target (native mode).
///
/// Syntax: `.JOBS: N target1 target2 ...`
/// Limits the named targets to at most N concurrent jobs.
void handleJobs()
{
import std.conv : to;
Target* jobsTarget = findTarget(".JOBS");
if (jobsTarget is null)
return;
if (jobsTarget.prerequisites.length < 2)
return;
size_t limit;
try
{
limit = jobsTarget.prerequisites[0].to!size_t;
}
catch (Exception)
{
return;
}
foreach (i, name; jobsTarget.prerequisites[1 .. $])
{
auto tp = findTarget(name);
if (tp !is null)
tp.jobLimit = limit;
}
jobsTarget.kind = TargetKind.phony;
phonyTargets[".JOBS"] = true;
}
/// Build reverse edges: populate `dependents` for each target.
/// For each target, scan all other targets' prerequisites and
/// add this target's name to the dependents list of each prereq.
void buildReverseEdges()
{
// Clear existing reverse edges.
foreach (ref t; targets)
t.dependents = [];
foreach (ref t; targets)
{
foreach (prereq; t.prerequisites ~ t.orderOnlyPrereqs)
{
auto tp = findTarget(prereq);
if (tp !is null)
tp.dependents ~= t.name;
}
}
}
/// Reset all scheduling state to defaults.
/// NOTE: jobLimit is NOT reset — it is set by handleJobs() and persists
/// across builds.
void resetSchedulingState()
{
foreach (ref t; targets)
{
t.state = BuildState.pending;
t.remainingDeps = 0;
t.criticalWeight = 0;
}
}
/// Compute remaining in-graph prerequisite count for each target.
/// Stores the count in each target's `remainingDeps` field.
/// Only counts prerequisites that exist as graph targets.
void computeRemainingDeps()
{
foreach (ref t; targets)
{
size_t count;
foreach (prereq; t.prerequisites ~ t.orderOnlyPrereqs)
{
if (hasTarget(prereq))
count++;
}
t.remainingDeps = count;
}
}
/// Compute the transitive closure of a root target.
/// Returns all target names reachable from root (including root itself).
string[] transitiveClosure(string root)
{
bool[string] visited;
string[] stack = [root];
string[] result;
while (stack.length > 0)
{
string current = stack[$ - 1];
stack = stack[0 .. $ - 1];
if (current in visited)
continue;
visited[current] = true;
result ~= current;
auto tp = findTarget(current);
if (tp is null)
continue;
foreach (prereq; tp.prerequisites ~ tp.orderOnlyPrereqs)
{
if (hasTarget(prereq) && prereq !in visited)
stack ~= prereq;
}
}
return result;
}
/// Detect cycles in the dependency graph using three-color DFS.
/// Stores found cycles as AntelopeError in cycleErrors.
void detectCycles()
source/antelope/build/output.d
+154
View File
@@ -0,0 +1,154 @@
/// Output buffering for parallel builds.
///
/// When multiple targets build concurrently, interleaved stdout/stderr
/// produces unreadable output. This module buffers each job's output
/// and prints it atomically when the job completes, keeping the build
/// log coherent even under full parallelism.
///
/// All methods are single-threaded — they're called exclusively from
/// the coordinator (main thread) after receiving results from workers.
/// Workers capture output locally and send it via JobDone messages;
/// no OutputManager access occurs in worker threads.
module antelope.build.output;
import std.stdio : writeln, stderr;
/// Manages per-target output buffering and atomic flush.
///
/// Two modes are supported:
/// `buffered` — Output is captured per target and printed atomically
/// when the target finishes building. This is the default
/// for parallel builds and produces clean, readable logs.
/// `live` — Output is printed immediately as it arrives. Multiple
/// concurrent targets will interleave their output
/// (GNU Make's default behaviour with `-j`).
class OutputManager
{
/// Whether to buffer output (true) or print live (false).
bool buffered = true;
private:
/// Per-target output buffers, keyed by target name.
string[][string] stdoutBufs;
string[][string] stderrBufs;
/// Map of target name → whether the target printed a non-@ line.
/// Used for GNU Make `-s` / `--silent` mode and @ prefix handling.
bool[string] hadEcho;
public:
/// Buffer a line of stdout for a target.
void bufferStdout(string targetName, string line)
{
stdoutBufs[targetName] ~= line;
}
/// Buffer a line of stderr for a target.
void bufferStderr(string targetName, string line)
{
stderrBufs[targetName] ~= line;
}
/// Record that the target printed (or would print) a command line.
/// Used to suppress "nothing to be done" messages when commands were echoed.
void markEchoed(string targetName)
{
hadEcho[targetName] = true;
}
/// Check whether the target echoed any command lines.
bool hasEchoed(string targetName)
{
return (targetName in hadEcho) !is null;
}
/// Print a line immediately without buffering (live mode fallback).
void printLive(string line)
{
writeln(line);
}
/// Print all buffered output for a completed target.
///
/// Stdout lines are printed first, then stderr.
/// Called by the coordinator after a worker reports completion.
void flush(string targetName)
{
string[] stdoutLines;
string[] stderrLines;
auto soPtr = targetName in stdoutBufs;
if (soPtr)
{
stdoutLines = *soPtr;
stdoutBufs.remove(targetName);
}
auto sePtr = targetName in stderrBufs;
if (sePtr)
{
stderrLines = *sePtr;
stderrBufs.remove(targetName);
}
if (stdoutLines.length == 0 && stderrLines.length == 0)
return;
foreach (line; stdoutLines)
writeln(line);
if (stderrLines.length > 0)
{
foreach (line; stderrLines)
stderr.writeln(line);
stderr.flush();
}
}
/// Clear all buffers without printing (used on failure cleanup).
void clearAll()
{
stdoutBufs = null;
stderrBufs = null;
hadEcho = null;
}
}
///
unittest
{
auto om = new OutputManager();
// Buffered mode: nothing printed during buffering
om.bufferStdout("foo.o", "cc -c foo.c");
om.bufferStdout("foo.o", "foo.c: In function 'main':");
om.bufferStderr("foo.o", "foo.c:5: warning: unused variable 'x'");
// Flush should produce all lines in order
om.flush("foo.o");
// After flush, buffers are empty — second flush is a no-op
om.flush("foo.o");
// Live mode: serialized printing
om.buffered = false;
om.printLive("live output line");
}
///
unittest
{
auto om = new OutputManager();
// Multiple targets interleaved buffering
om.bufferStdout("a", "building a");
om.bufferStdout("b", "building b");
om.bufferStdout("a", "a done");
om.flush("a");
om.flush("b");
// echo tracking
om.markEchoed("c");
assert(om.hasEchoed("c"));
assert(!om.hasEchoed("nonexistent"));
}
source/antelope/build/pool.d
+922
View File
@@ -0,0 +1,922 @@
/// Parallel build worker pool with dependency-aware scheduling.
///
/// Uses D's `std.concurrency` Actor model for worker coordination:
/// - Workers are spawned as OS threads via `spawn()`
/// - The main thread acts as coordinator: maintains the ready queue,
/// dispatches BuildJob messages to workers, and receives JobDone results
/// - No shared mutable state — all communication is via message passing
///
/// Features:
/// - Ready queue with critical-path priority sorting (load-aware scheduling)
/// - Fine-grained dispatch: targets become ready immediately when their
/// last prerequisite completes (not batched)
/// - Worker pool reuse: N threads spawned once, reused for all targets
/// - Configurable job limits (-jN)
/// - Correct failure propagation: failed targets mark dependents as skipped
/// - .NOTPARALLEL support: targets run exclusively when all workers idle
/// - Output buffering via OutputManager for atomic per-target printing
module antelope.build.pool;
import core.thread : Thread;
import std.concurrency : spawn, send, receive, receiveOnly, receiveTimeout,
Tid, thisTid, ownerTid;
import std.algorithm : sort;
import std.conv : to;
import antelope.build.target;
import antelope.build.graph;
import antelope.build.dependency;
import antelope.build.output;
import antelope.build.executor;
import antelope.shell.process;
import antelope.shell.environment;
import antelope.filesystem.timestamps;
import antelope.compatibility.parallel;
import antelope.compatibility.vpath;
import antelope.diagnostics.output;
// ── Messages ────────────────────────────────────────────────────────────
/// Sent from coordinator to worker: "build this target."
struct BuildJob
{
string targetName; /// Target to build
immutable(string)[] expandedRecipe; /// Pre-expanded recipe lines
immutable(bool)[] ignoreErrors; /// Per-line: - prefix
immutable(bool)[] silent; /// Per-line: @ prefix
immutable(string)[] execEnv; /// KEY=VALUE environment
bool dryRun; /// If true, echo but don't execute
}
/// Sent from worker to coordinator: "target built (or failed)."
struct JobDone
{
Tid workerTid; /// Which worker completed
string targetName; /// Which target was built
bool success; /// True if all recipe lines succeeded
int exitCode; /// Last non-zero exit code (0 on success)
bool hadEcho; /// True if any recipe line was echoed (non-@)
immutable(string)[] stdoutLines; /// Captured stdout lines
immutable(string)[] stderrLines; /// Captured stderr lines
}
/// Sent from coordinator to worker: "exit your loop."
struct Shutdown {}
// ── WorkerPool ───────────────────────────────────────────────────────────
/// Manages parallel build execution.
///
/// Usage:
/// ```d
/// auto pool = WorkerPool(config.jobs);
/// int exitCode = pool.build(graph, rootTargets, config, env, expander, output, vpath);
/// ```
struct WorkerPool
{
private:
uint numWorkers; /// Number of worker threads (= -j value)
Tid[] workerTids; /// Tids of spawned workers
bool started; /// Whether workers have been spawned
// Build state (populated during build())
DependencyGraph* graph;
ParallelConfig* parallelConfig;
OutputManager* outputMgr;
VPathConfig* vpathConfig; /// VPATH for needsRebuild in dequeueDependents
bool[string] notParallelSet; /// Targets marked .NOTPARALLEL
bool[string] failedSet; /// Targets that failed (for propagation)
bool[string] skippedSet; /// Targets blocked by failed prereqs
// Ready queue (sorted by descending criticalWeight)
// Stored as indices into graph.targets
size_t[] readyQueue;
// Dependency tracking
size_t[string] remainingDeps;
// .JOBS throttle: tracks active workers per job-limit value.
size_t[size_t] activeByLimit;
// Completed target counter (shared between build() and dequeueDependents).
size_t completedCount;
public:
/// Create a worker pool with the given number of workers.
///
/// If `numWorkers` is 0, it defaults to the number of CPU cores.
/// If `numWorkers` is 1, the build runs serially without spawning threads.
static WorkerPool create(uint numWorkers = 0)
{
import std.parallelism : totalCPUs;
WorkerPool pool;
if (numWorkers == 0)
pool.numWorkers = totalCPUs;
else
pool.numWorkers = numWorkers;
return pool;
}
/// Jobserver file descriptors for cross-process token coordination.
/// readFd is passed to child processes via MAKEFLAGS; writeFd is
/// held by the parent to return tokens after job completion.
struct JobserverPipe
{
int readFd = -1; /// Read end — child processes consume tokens here
int writeFd = -1; /// Write end — parent writes tokens back on completion
bool active; /// Whether the jobserver is operational
}
/// Create a jobserver pipe with `nTokens` initial tokens.
///
/// Writes N bytes to the pipe so that up to N jobs can run
/// concurrently across recursive $(MAKE) invocations. Each job
/// reads one byte before starting; the byte is written back on
/// completion.
static JobserverPipe createJobserverPipe(uint nTokens)
{
version (Posix)
{
import core.sys.posix.unistd : pipe, read, write, close;
import core.sys.posix.fcntl : fcntl, F_SETFD, FD_CLOEXEC;
JobserverPipe js;
int[2] fds;
if (pipe(fds) != 0)
return js;
// Write end: mark close-on-exec so child processes only
// inherit the read end.
fcntl(fds[1], F_SETFD, FD_CLOEXEC);
js.readFd = fds[0];
js.writeFd = fds[1];
js.active = true;
// Seed the pipe with N tokens (one byte each).
ubyte token = 0;
for (uint i = 0; i < nTokens; i++)
write(js.writeFd, &token, 1);
return js;
}
else
{
// Non-POSIX: jobserver not supported.
return JobserverPipe();
}
}
/// Consume one token from the jobserver pipe (blocking).
/// Returns true if a token was acquired, false on error.
private static bool acquireJobserverToken(int readFd)
{
version (Posix)
{
import core.sys.posix.unistd : read;
ubyte token;
return read(readFd, &token, 1) == 1;
}
else
return false;
}
/// Return one token to the jobserver pipe.
private static bool releaseJobserverToken(int writeFd)
{
version (Posix)
{
import core.sys.posix.unistd : write;
ubyte token = 0;
return write(writeFd, &token, 1) == 1;
}
else
return false;
}
/// Run the build for the given root targets.
///
/// Params:
/// graph = Dependency graph with all targets (mutated: runtime state fields)
/// roots = Root target names to build (e.g., ["all"])
/// config = Parallel config (jobs, notParallelTargets, output sync mode)
/// env = Build environment (passed to recipe subprocesses)
/// expander = Variable expansion delegate
/// output = Output buffer manager
/// vpath = VPATH config (for needsRebuild checks)
/// baseExecEnv = KEY=VALUE pairs added to every job's env (e.g., SHELL, MAKEFLAGS)
/// dryRun = Print commands without executing
/// silent = Suppress command echoing
///
/// Returns: exit code (0 = success, non-zero = failure).
int build(
ref DependencyGraph graph,
string[] roots,
ref ParallelConfig config,
Environment* env,
string delegate(string, string, string[], string) expander,
OutputManager* output,
VPathConfig* vpath,
string[] baseExecEnv = [],
bool dryRun = false,
bool silent = false)
{
import antelope.filesystem.timestamps;
this.graph = &graph;
this.parallelConfig = &config;
this.outputMgr = output;
this.vpathConfig = vpath;
// Determine actual worker count.
uint nWorkers = config.jobs;
if (nWorkers == 0)
{
import std.parallelism : totalCPUs;
nWorkers = totalCPUs;
}
// Reset per-build state.
readyQueue = [];
failedSet = null;
skippedSet = null;
remainingDeps = null;
// Serial mode shortcut.
if (nWorkers <= 1)
return buildSerial(graph, roots, config, env, expander, output,
vpath, baseExecEnv, dryRun, silent);
// Build the combined transitive closure for all root targets.
bool[string] inClosure;
foreach (root; roots)
{
auto closure = graph.transitiveClosure(root);
foreach (name; closure)
inClosure[name] = true;
}
if (inClosure.length == 0)
return 0;
// Set up scheduling state.
graph.resetSchedulingState();
graph.buildReverseEdges();
import antelope.build.dependency;
foreach (root; roots)
computeCriticalWeights(graph, root);
graph.computeRemainingDeps();
// Copy remainingDeps for fast lookup BEFORE the init loop so
// that dequeueDependents (called for up-to-date targets during
// initial ready-queue construction) can read from it.
remainingDeps = null;
foreach (ref t; graph.targets)
if (t.name in inClosure)
remainingDeps[t.name] = t.remainingDeps;
// Populate initial ready queue: targets with remainingDeps == 0
// that actually need building and are in the closure.
size_t[] initialReady;
foreach (i, ref t; graph.targets)
{
if (t.name !in inClosure)
continue;
if (t.remainingDeps != 0)
continue;
if (!needsRebuild(t.name, t.prerequisites,
&graph.phonyTargets, vpath, &t.orderOnlyPrereqs))
{
t.state = BuildState.completed;
// Notify dependents so they can become ready.
dequeueDependents(t.name);
continue;
}
initialReady ~= i;
}
// Sort initial-ready targets by descending critical weight.
initialReady.sort!((a, b) =>
graph.targets[a].criticalWeight > graph.targets[b].criticalWeight);
// Merge initial-ready targets with any targets that became ready
// during the init loop (via dequeueDependents for up-to-date
// prerequisites). Initial-ready goes first (already filtered
// by needsRebuild), then dequeueDependents-added targets.
readyQueue = initialReady ~ readyQueue;
size_t totalTargets = 0;
foreach (i, ref t; graph.targets)
if (t.name in inClosure && t.state != BuildState.completed)
totalTargets++;
// Spawn worker threads.
workerTids.length = 0;
for (uint i = 0; i < nWorkers; i++)
{
auto tid = spawn(&workerFunc);
workerTids ~= tid;
}
this.numWorkers = nWorkers;
this.started = true;
// Seed idle worker queue: all workers start idle.
Tid[] idleWorkers = workerTids.dup;
// Send initial batch of jobs (respect .JOBS and .NOTPARALLEL limits).
completedCount = 0;
while (readyQueue.length > 0 && idleWorkers.length > 0)
{
size_t idx = readyQueue[0];
// NOTPARALLEL: only dispatch when all other workers are idle.
if (idx < graph.targets.length &&
graph.targets[idx].name in notParallelSet &&
idleWorkers.length != workerTids.length)
break;
// .JOBS limit: throttle if this target has a job limit.
if (idx < graph.targets.length &&
graph.targets[idx].jobLimit > 0)
{
size_t busy = workerTids.length - idleWorkers.length;
if (busy >= graph.targets[idx].jobLimit)
break;
}
readyQueue = readyQueue[1 .. $];
auto job = makeBuildJob(idx, env, expander, baseExecEnv, dryRun);
if (job.expandedRecipe.length > 0)
{
graph.targets[idx].state = BuildState.running;
send(idleWorkers[$ - 1], job);
idleWorkers = idleWorkers[0 .. $ - 1];
if (graph.targets[idx].jobLimit > 0)
activeByLimit[graph.targets[idx].jobLimit]++;
}
else
{
// Target with no recipe: mark complete immediately.
graph.targets[idx].state = BuildState.completed;
dequeueDependents(graph.targets[idx].name);
completedCount++;
}
}
// Coordinator loop: dispatch → receive → process → repeat.
bool hasFailure = false;
while (completedCount < totalTargets)
{
// Phase 1: Dispatch as many ready targets as possible
// to idle workers.
while (readyQueue.length > 0 && idleWorkers.length > 0)
{
size_t idx = readyQueue[0];
// NOTPARALLEL: only dispatch when all other workers are idle.
if (idx < graph.targets.length &&
graph.targets[idx].name in notParallelSet &&
idleWorkers.length != workerTids.length)
break;
// .JOBS limit: throttle if this target has a job limit.
// Tracks only workers running .JOBS-limited targets of
// the same limit value, not total busy workers.
if (idx < graph.targets.length &&
graph.targets[idx].jobLimit > 0)
{
size_t limit = graph.targets[idx].jobLimit;
auto countPtr = limit in activeByLimit;
size_t active = countPtr ? *countPtr : 0;
if (active >= limit)
break;
}
readyQueue = readyQueue[1 .. $];
auto job = makeBuildJob(idx, env, expander, baseExecEnv, dryRun);
if (job.expandedRecipe.length > 0)
{
graph.targets[idx].state = BuildState.running;
send(idleWorkers[$ - 1], job);
idleWorkers = idleWorkers[0 .. $ - 1];
if (graph.targets[idx].jobLimit > 0)
activeByLimit[graph.targets[idx].jobLimit]++;
}
else
{
// Target with no recipe: complete immediately.
graph.targets[idx].state = BuildState.completed;
dequeueDependents(graph.targets[idx].name);
completedCount++;
}
}
// Phase 2: Check termination.
if (idleWorkers.length == workerTids.length)
{
// All workers idle. If work remains, targets are
// blocked (waiting for failed/skipped prereqs).
if (completedCount < totalTargets)
{
foreach (ref t; graph.targets)
{
if (t.name !in inClosure)
continue;
if (t.state == BuildState.pending)
{
t.state = BuildState.skipped;
skippedSet[t.name] = true;
completedCount++;
if (output && output.buffered)
output.flush(t.name);
}
}
}
break;
}
// Phase 3: Wait for a worker result.
auto done = receiveOnly!JobDone();
// Worker is now idle.
idleWorkers ~= done.workerTid;
// Buffer output into OutputManager.
if (output)
{
foreach (line; done.stdoutLines)
output.bufferStdout(done.targetName, cast(string) line);
foreach (line; done.stderrLines)
output.bufferStderr(done.targetName, cast(string) line);
if (done.hadEcho)
output.markEchoed(done.targetName);
}
// Phase 4: Process result.
auto tp = graph.findTarget(done.targetName);
if (tp !is null)
{
// Decrement .JOBS throttle counter if this target had a limit.
if (tp.jobLimit > 0)
{
auto countPtr = tp.jobLimit in activeByLimit;
if (countPtr && *countPtr > 0)
(*countPtr)--;
}
if (done.success)
{
tp.state = BuildState.completed;
log(LogLevel.dbg, "[" ~ done.targetName ~ "] completed");
dequeueDependents(done.targetName);
}
else
{
tp.state = BuildState.failed;
failedSet[done.targetName] = true;
hasFailure = true;
log(LogLevel.dbg, "[" ~ done.targetName ~
"] FAILED (exit " ~ done.exitCode.to!string ~ ")");
propagateFailure(done.targetName, inClosure);
}
}
// Flush buffered output.
if (output && output.buffered)
output.flush(done.targetName);
completedCount++;
// Loop back to Phase 1 (dispatch newly-ready targets).
}
// Shutdown all workers.
foreach (tid; workerTids)
{
try { send(tid, Shutdown()); } catch (Exception) {}
}
// Collect any late messages (workers may have sent results
// that haven't been received yet due to timing).
// Use a short timeout to drain the queue.
import core.time : dur;
while (true)
{
auto msg = receiveTimeout(dur!"msecs"(100), (JobDone d) => true);
if (!msg)
break;
}
workerTids = [];
started = false;
return hasFailure ? 1 : 0;
}
private:
/// Serial fallback for -j1 or single-worker builds.
int buildSerial(
ref DependencyGraph graph,
string[] roots,
ref ParallelConfig config,
Environment* env,
string delegate(string, string, string[], string) expander,
OutputManager* output,
VPathConfig* vpath,
string[] baseExecEnv,
bool dryRun,
bool silent)
{
import antelope.filesystem.timestamps : needsRebuild;
bool hasFailure;
foreach (root; roots)
{
auto batches = resolveDependencies(graph, root);
foreach (batch; batches)
{
foreach (ref t; batch)
{
if (!needsRebuild(t.name, t.prerequisites,
&graph.phonyTargets, vpath, &t.orderOnlyPrereqs))
continue;
string[] execEnv = baseExecEnv.dup;
auto result = executeTarget(t, execEnv, expander,
output, dryRun, silent);
if (output && output.buffered)
output.flush(t.name);
if (!result.success)
{
hasFailure = true;
goto done;
}
}
}
}
done:
return hasFailure ? 1 : 0;
}
/// Worker thread function.
static void workerFunc()
{
import std.string : stripLeft;
bool running = true;
while (running)
{
receive(
(BuildJob job) {
JobDone done;
done.workerTid = thisTid;
done.targetName = job.targetName;
done.success = true;
done.exitCode = 0;
done.hadEcho = false;
// Build mutable output buffers, freeze before sending.
string[] outLines;
string[] errLines;
if (job.expandedRecipe.length == 0)
{
done.stdoutLines = outLines.idup;
done.stderrLines = errLines.idup;
send(ownerTid, done);
return;
}
foreach (i, line; job.expandedRecipe)
{
// line is immutable(string); cast to string for stdlib.
string sline = cast(string) line;
if (sline.stripLeft.length == 0)
continue;
bool ignoreErrors = i < job.ignoreErrors.length
? cast(bool) job.ignoreErrors[i] : false;
bool silent = i < job.silent.length
? cast(bool) job.silent[i] : false;
// Echo
if (!silent)
{
done.hadEcho = true;
outLines ~= sline;
}
// Dry run: skip actual execution.
if (job.dryRun)
continue;
// Execute. Cast execEnv back to mutable for runProcessPiped.
auto ph = runProcessPiped(sline, cast(string[]) job.execEnv);
// Read pipes.
try
{
import std.string : chomp;
foreach (pl; ph.stdoutPipe.byLine)
{
string s = pl.chomp().idup;
outLines ~= s;
}
foreach (pl; ph.stderrPipe.byLine)
{
string s = pl.chomp().idup;
errLines ~= s;
}
}
catch (Exception e)
{
// Log pipe read errors but don't abort the build.
log(LogLevel.dbg, "[" ~ job.targetName ~
"] pipe read error: " ~ e.msg);
}
int code = ph.waitFor();
if (code != 0 && !ignoreErrors)
{
ph.closePipes();
done.success = false;
done.exitCode = code;
done.stdoutLines = outLines.idup;
done.stderrLines = errLines.idup;
send(ownerTid, done);
return;
}
ph.closePipes();
}
done.stdoutLines = outLines.idup;
done.stderrLines = errLines.idup;
send(ownerTid, done);
},
(Shutdown _) {
running = false;
}
);
}
}
/// Create a BuildJob for the target at graph index `idx`.
BuildJob makeBuildJob(
size_t idx,
Environment* env,
string delegate(string, string, string[], string) expander,
string[] baseExecEnv,
bool dryRun)
{
auto t = &graph.targets[idx];
// Build mutable arrays, then freeze to immutable for sending.
string[] recipeLines;
bool[] ignoreErrs;
bool[] silents;
foreach (line; t.recipe)
{
// Expand variables in the recipe.
string expanded = expander(line, t.name,
t.prerequisites, t.stem);
// Strip and classify prefix characters.
import std.string : stripLeft;
string trimmed = expanded.stripLeft();
bool ignoreErrors;
bool silent;
if (trimmed.length > 0)
{
bool stripping = true;
while (stripping && trimmed.length > 0)
{
stripping = false;
switch (trimmed[0])
{
case '@':
silent = true;
trimmed = trimmed[1 .. $];
stripping = true;
break;
case '-':
ignoreErrors = true;
trimmed = trimmed[1 .. $];
stripping = true;
break;
case '+':
trimmed = trimmed[1 .. $];
stripping = true;
break;
default:
break;
}
}
}
if (trimmed.length == 0)
continue;
recipeLines ~= trimmed;
ignoreErrs ~= ignoreErrors;
silents ~= silent;
}
// Build execEnv.
string[] execEnvArr = baseExecEnv.dup;
// Freeze arrays to immutable for std.concurrency message passing.
BuildJob job;
job.targetName = t.name;
job.expandedRecipe = recipeLines.idup;
job.ignoreErrors = ignoreErrs.idup;
job.silent = silents.idup;
job.execEnv = execEnvArr.idup;
job.dryRun = dryRun;
return job;
}
/// Decrement remainingDeps for all dependents of `completedTarget`.
/// Any dependent that reaches 0 remaining deps is added to the
/// ready queue (sorted by descending critical weight).
void dequeueDependents(string completedTarget)
{
auto tp = graph.findTarget(completedTarget);
if (tp is null)
return;
foreach (depName; tp.dependents)
{
// Skip if not in our tracking (could be outside closure).
auto depPtr = depName in remainingDeps;
if (depPtr is null)
continue;
if (*depPtr == 0)
continue; // Already ready or completed
(*depPtr)--;
if (*depPtr == 0 && !(depName in skippedSet))
{
// Target is now ready — check if it needs building.
auto dep = graph.findTarget(depName);
if (dep is null || dep.state != BuildState.pending)
continue;
// Check up-to-date: targets that become ready via
// dequeueDependents were NOT filtered during the init
// loop (which only checks initially-zero-dep targets).
import antelope.filesystem.timestamps : needsRebuild;
if (!needsRebuild(dep.name, dep.prerequisites,
&graph.phonyTargets, vpathConfig, &dep.orderOnlyPrereqs))
{
dep.state = BuildState.completed;
completedCount++;
dequeueDependents(dep.name);
continue;
}
// Insert sorted by descending critical weight.
bool inserted;
foreach (i, qi; readyQueue)
{
if (dep.criticalWeight > graph.targets[qi].criticalWeight)
{
readyQueue = readyQueue[0 .. i] ~
[cast(size_t)(dep - graph.targets.ptr)] ~
readyQueue[i .. $];
inserted = true;
break;
}
}
if (!inserted)
readyQueue ~= cast(size_t)(dep - graph.targets.ptr);
}
}
}
/// Mark all dependents of a failed target as skipped.
/// Recursively propagates: if A depends on B and B fails,
/// A is skipped; if C depends on A, C is also skipped.
void propagateFailure(string failedTarget, ref bool[string] inClosure)
{
import std.algorithm : canFind;
string[] stack = [failedTarget];
while (stack.length > 0)
{
string current = stack[$ - 1];
stack = stack[0 .. $ - 1];
auto tp = graph.findTarget(current);
if (tp is null)
continue;
foreach (depName; tp.dependents)
{
if (depName in skippedSet || depName in failedSet)
continue;
if (depName !in inClosure)
continue;
auto dep = graph.findTarget(depName);
if (dep is null)
continue;
dep.state = BuildState.skipped;
skippedSet[depName] = true;
stack ~= depName;
}
}
}
}
// ── Unittests ────────────────────────────────────────────────────────────
///
unittest
{
// Build a trivial graph with one target (no recipe).
DependencyGraph g;
g.addTarget(Target("leaf", TargetKind.file, [], []));
g.buildReverseEdges();
g.computeRemainingDeps();
computeCriticalWeights(g, "leaf");
ParallelConfig pc;
pc.jobs = 2;
auto om = new OutputManager();
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto pool = WorkerPool.create(2);
int code = pool.build(g, ["leaf"], pc, null, &expand, &om, null);
assert(code == 0);
assert(g.findTarget("leaf").state == BuildState.completed);
}
///
unittest
{
// Chain: a → b → c (all no recipe, always succeed).
DependencyGraph g;
g.addTarget(Target("c", TargetKind.file, [], []));
g.addTarget(Target("b", TargetKind.file, ["c"], []));
g.addTarget(Target("a", TargetKind.file, ["b"], []));
// Ensure all are in the closure.
g.buildReverseEdges();
g.computeRemainingDeps();
computeCriticalWeights(g, "a");
ParallelConfig pc;
pc.jobs = 2;
auto om = new OutputManager();
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto pool = WorkerPool.create(2);
int code = pool.build(g, ["a"], pc, null, &expand, &om, null);
assert(code == 0);
assert(g.findTarget("a").state == BuildState.completed);
assert(g.findTarget("b").state == BuildState.completed);
assert(g.findTarget("c").state == BuildState.completed);
}
// Regression: serial mode (jobs=1) should work.
unittest
{
DependencyGraph g;
g.addTarget(Target("x", TargetKind.file, [], []));
g.buildReverseEdges();
g.computeRemainingDeps();
computeCriticalWeights(g, "x");
ParallelConfig pc;
pc.jobs = 1;
auto om = new OutputManager();
string expand(string ln, string tn, string[] pr, string st) { return ln; }
auto pool = WorkerPool.create(1);
int code = pool.build(g, ["x"], pc, null, &expand, &om, null);
assert(code == 0);
}
source/antelope/build/target.d
+43
View File
@@ -9,6 +9,20 @@ enum TargetKind
intermediate,
}
/// Runtime scheduling state for parallel builds.
///
/// Tracks where a target is in the build lifecycle so the worker pool
/// can make dispatch decisions.
enum BuildState
{
pending, /// Not yet ready — outstanding in-graph prerequisites
ready, /// All prereqs satisfied, can be dispatched to a worker
running, /// Currently being built by a worker thread
completed, /// Built successfully
failed, /// Build failed — dependents will be skipped
skipped, /// Blocked by a failed prerequisite
}
/// A single build target.
struct Target
{
@@ -18,4 +32,33 @@ struct Target
string[] recipe; /// Shell commands to build this target
string[] orderOnlyPrereqs; /// Order-only prerequisites (| — must exist, no rebuild trigger)
string stem; /// Pattern/suffix rule stem for $* expansion
// ── Runtime scheduling fields ───────────────────────────────────────
/// Current build state (mutated by the worker pool coordinator).
BuildState state = BuildState.pending;
/// How many in-graph prerequisites still need to complete before this
/// target becomes `ready`. Set by `DependencyGraph.computeRemainingDeps()`
/// before the build begins.
size_t remainingDeps;
/// Critical-path weight for load-aware scheduling.
///
/// Computed as: `recipe.length + max(successor.criticalWeight)`.
/// Leaf nodes (no in-graph successors) have weight = `recipe.length`.
/// Higher weight → on the critical path → should be scheduled first
/// when multiple targets are in the ready queue.
size_t criticalWeight;
/// Reverse edges: names of targets that list this target as a prerequisite.
/// Populated by `DependencyGraph.buildReverseEdges()` before scheduling.
/// When this target completes, every name in this list will have its
/// `remainingDeps` decremented.
string[] dependents;
/// Per-target job limit for .JOBS special target (native mode only).
/// 0 = use the global pool limit. Non-zero = maximum concurrent
/// jobs allowed for this specific target's recipe group.
size_t jobLimit;
}
source/antelope/cli/subcommands.d
+105 -79
View File
@@ -29,6 +29,8 @@ int dispatchSubcommand(CliConfig config)
/// Execute the build (default subcommand).
///
/// Full pipeline: find build file → parse → evaluate → schedule → execute.
/// The schedule/execute phases now use the parallel WorkerPool for
/// dependency-aware concurrent builds when `-j > 1`.
int runBuild(CliConfig config)
{
import antelope.parser.parser;
@@ -40,8 +42,12 @@ int runBuild(CliConfig config)
import antelope.build.dependency;
import antelope.build.scheduler;
import antelope.build.executor;
import antelope.build.pool;
import antelope.build.output;
import antelope.shell.environment;
import antelope.filesystem.timestamps;
import antelope.compatibility.parallel;
import antelope.compatibility.submake;
// Set log level
if (config.debugMode)
@@ -104,10 +110,12 @@ int runBuild(CliConfig config)
// Set MAKE to the antelope binary path for $(MAKE) in recipes.
// Include -gnu so recursive sub-makes inherit GNU compat mode.
// config.file is shell-quoted to prevent injection when $(MAKE)
// is used in recipes.
import std.file : thisExePath;
string makeCmd = thisExePath();
if (config.gnuMode) makeCmd ~= " -gnu";
if (config.file.length > 0) makeCmd ~= " -f " ~ config.file;
if (config.file.length > 0) makeCmd ~= " -f '" ~ escapeShell(config.file) ~ "'";
env.set("MAKE", makeCmd);
// Set MAKECMDGOALS from command-line targets (autotools compat)
@@ -171,7 +179,13 @@ int runBuild(CliConfig config)
return 1;
}
// Process special targets after evaluation populates the graph.
(*graph).handlePhony();
(*graph).handleWait();
(*graph).handleJobs();
// Check for cycle errors
(*graph).detectCycles();
if (graph.cycleErrors.length > 0)
{
foreach (err; graph.cycleErrors)
@@ -217,7 +231,7 @@ int runBuild(CliConfig config)
else
buildTargets = [graph.targets[0].name];
// .PHONY targets are tracked in the graph automatically via handlePhony()
// .PHONY + .WAIT + .JOBS targets are processed above via handlePhony/handleWait/handleJobs.
// --- VPATH configuration (GNU Make compat) ---
import antelope.compatibility.vpath;
@@ -252,16 +266,14 @@ int runBuild(CliConfig config)
}
}
// Build each requested target
int exitCode = 0;
// --- Resolve implicit targets (GNU Make compat) ---
// Targets requested on the command line might not exist in the graph
// yet — they may be defined only via pattern/suffix rules.
// We create stubs and run implicit rule resolution so they can be built.
foreach (targetName; buildTargets)
{
if (!graph.hasTarget(targetName))
{
// Target not explicitly defined — try to create it from
// implicit rules (suffix rules, pattern rules). Autotools
// Makefiles invoke $(MAKE) with targets like "be.gmo" that
// are only defined via suffix rules (e.g., .po.gmo:).
import antelope.build.target;
Target stub;
stub.name = targetName;
@@ -270,93 +282,107 @@ int runBuild(CliConfig config)
import antelope.evaluator.evaluator : resolveImplicitRules;
resolveImplicitRules(*graph, env);
if (!graph.hasTarget(targetName) ||
graph.findTarget(targetName).recipe.length == 0)
{
log(LogLevel.normal, "antelope: *** No rule to make target '" ~
targetName ~ "'. Stop.");
return 1;
}
}
}
// Resolve dependencies and check what needs building
auto batches = resolveDependencies(*graph, targetName);
// --- Set up parallel execution config ---
ParallelConfig parallelCfg;
parallelCfg.jobs = config.jobs;
parallelCfg.heuristic = SchedulingHeuristic.criticalPath;
import std.stdio;
if (targetName == "libgnu.a" || targetName == "all") {
stderr.writefln(" batches=%d", batches.length);
foreach (i, batch; batches) {
size_t w;
foreach (ref t; batch) if (t.recipe.length > 0) w++;
stderr.writefln(" batch[%d]: %d targets, %d with recipe", i, batch.length, w);
}
}
bool builtSomething = false;
foreach (batch; batches)
// Check for .NOTPARALLEL targets in the graph.
if (graph.hasTarget(".NOTPARALLEL"))
{
auto np = graph.findTarget(".NOTPARALLEL");
if (np !is null)
{
foreach (ref t; batch)
foreach (name; np.prerequisites)
parallelCfg.notParallelTargets ~= name;
}
}
// Output mode: buffer when parallel, live when serial.
auto outputMgr = new OutputManager();
if (config.jobs > 1 || config.jobs == 0)
{
outputMgr.buffered = true;
parallelCfg.outputSync = OutputSyncMode.target;
}
// --- Build base execution environment ---
string[] baseExecEnv;
if (env.hasKey("SHELL"))
baseExecEnv ~= "SHELL=" ~ env.get("SHELL");
// Create jobserver pipe for cross-process token coordination.
// Only created when parallel build is active (-j > 1).
// Create jobserver pipe for cross-process token coordination.
// Only created when parallel build is active (-j > 1).
import antelope.build.pool : WorkerPool;
WorkerPool.JobserverPipe jsPipe;
if (config.jobs > 1)
jsPipe = WorkerPool.createJobserverPipe(config.jobs);
// Serialize MAKEFLAGS for recursive $(MAKE) calls.
string makeFlags = serializeMakeFlags(config, jsPipe.readFd, jsPipe.writeFd);
if (makeFlags.length > 0)
baseExecEnv ~= "MAKEFLAGS=" ~ makeFlags;
// --- Variable expansion delegate ---
// Captured by the pool and called per-target during job construction.
// This delegates to the existing expand() function from the evaluator,
// threading through the global Environment and target context.
string expander(string line, string targetName,
string[] prerequisites, string stem)
{
return expand(line, env, targetName, prerequisites, stem);
}
// --- Dispatch build ---
auto pool = WorkerPool.create(config.jobs);
int exitCode = pool.build(
*graph, buildTargets, parallelCfg, env,
&expander, &outputMgr, &vpath,
baseExecEnv, config.dryRun, false);
// Report up-to-date targets (tracks targets that had no work).
if (exitCode == 0)
{
bool anyBuilt;
foreach (targetName; buildTargets)
{
auto tp = graph.findTarget(targetName);
if (tp !is null && tp.state == BuildState.completed)
{
if (!needsRebuild(t.name, t.prerequisites,
&graph.phonyTargets, &vpath, &t.orderOnlyPrereqs))
continue;
builtSomething = true;
// Execute recipe lines
foreach (recipeLine; t.recipe)
if (tp.recipe.length == 0 && !outputMgr.hasEchoed(targetName))
{
// Expand variables in the recipe
string expanded = expand(recipeLine, env, t.name,
t.prerequisites, t.stem);
// Print the command unless silent (@ prefix)
import std.string : stripLeft;
string trimmed = recipeLine.stripLeft();
if (config.dryRun || config.debugMode || recipeLine.length == 0 ||
(trimmed.length > 0 && trimmed[0] != '@'))
{
log(LogLevel.normal, expanded);
}
// Build environment for recipe execution
// (propagate SHELL from Makefile if set)
string[] execEnv;
if (env.hasKey("SHELL"))
execEnv ~= "SHELL=" ~ env.get("SHELL");
// Serialize MAKEFLAGS for recursive $(MAKE) calls
import antelope.compatibility.submake;
string makeFlags = serializeMakeFlags(config);
if (makeFlags.length > 0)
execEnv ~= "MAKEFLAGS=" ~ makeFlags;
// Execute unless dry run
if (!config.dryRun)
{
auto result = execute(expanded, execEnv);
if (!result.success)
{
log(LogLevel.normal, "antelope: *** [" ~ t.name ~
"] Error " ~ result.exitCode.to!string);
return result.exitCode;
}
}
// Target was up-to-date or had no recipe.
}
else if (!outputMgr.hasEchoed(targetName))
{
log(LogLevel.normal, "antelope: '" ~ targetName ~
"' is up to date.");
}
anyBuilt = true;
}
}
if (!builtSomething)
if (!anyBuilt)
{
log(LogLevel.normal, "antelope: '" ~ targetName ~
"' is up to date.");
// Check if nothing needed building (all targets already up to date).
}
}
return exitCode;
}
/// Escape a string for single-quoted shell usage.
/// Replaces each `'` with `'\''` so the value can be wrapped in single quotes.
private string escapeShell(string s)
{
import std.array : replace;
return s.replace("'", "'\\''");
}
/// Find which build file to use based on mode and config.
private string findBuildFile(CliConfig config)
{
source/antelope/compatibility/parallel.d
+33 -1
View File
@@ -18,13 +18,45 @@ enum ParallelSpecialTarget
jobs, /// .JOBS
}
/// Output synchronisation mode for parallel builds.
///
/// GNU Make 4.0+ supports `--output-sync` with four modes.
enum OutputSyncMode
{
none, /// No synchronisation — output may be interleaved (default)
target, /// Buffer output per target, print atomically on completion
line, /// Buffer output per line, print atomically per line
recurse, /// Buffer output per recursive make invocation
}
/// Which scheduling heuristic to use when multiple targets are ready.
enum SchedulingHeuristic
{
fifo, /// First-in-first-out — build order is topological order
criticalPath, /// Prioritise targets on the critical path (minimise total build time)
}
/// Parallel execution configuration.
struct ParallelConfig
{
/// Maximum parallel jobs (0 = unlimited, 1 = serial).
uint jobs = 1;
/// Targets excluded from parallel builds.
/// Targets excluded from parallel builds (.NOTPARALLEL).
/// These always run serially, even when -j > 1.
string[] notParallelTargets;
/// Whether to use jobserver protocol for sub-makes.
bool useJobserver = true;
/// How to synchronise output from parallel jobs.
OutputSyncMode outputSync = OutputSyncMode.none;
/// Which heuristic to use for ordering the ready queue.
SchedulingHeuristic heuristic = SchedulingHeuristic.criticalPath;
/// Timeout in seconds for individual recipe lines (0 = no timeout).
/// GNU Make does not natively support job timeouts; this is an
/// Antelope extension.
uint jobTimeoutSecs = 0;
}
source/antelope/compatibility/submake.d
+30 -6
View File
@@ -14,7 +14,6 @@ module antelope.compatibility.submake;
import antelope.cli.args;
import std.conv : to;
import std.string : strip;
import std.string : strip;
/// Sub-make communication options.
struct SubMakeConfig
@@ -36,14 +35,30 @@ struct SubMakeConfig
/// MAKEFLAGS environment variable for recursive $(MAKE) invocations.
///
/// Serialized flags:
/// -j<N> — parallel job count (only when jobs > 1)
/// -n — dry run
/// -P — POSIX conformance mode
/// -d — debug output
/// VAR=val — command-line variable overrides
/// -j<N> — parallel job count (only when jobs > 1)
/// -n — dry run
/// -P — POSIX conformance mode
/// -d — debug output
/// --jobserver-auth=R,W — jobserver pipe file descriptors (when provided)
/// VAR=val — command-line variable overrides
///
/// Returns: a space-delimited MAKEFLAGS string, or "" if no flags are active.
string serializeMakeFlags(CliConfig config)
{
return serializeMakeFlagsImpl(config, 0, 0);
}
/// Serialize MAKEFLAGS including jobserver pipe descriptors.
///
/// When `readFd` and `writeFd` are non-zero, appends
/// `--jobserver-auth=<readFd>,<writeFd>` to the MAKEFLAGS string
/// so sub-make processes can participate in the shared job pool.
string serializeMakeFlags(CliConfig config, int readFd, int writeFd)
{
return serializeMakeFlagsImpl(config, readFd, writeFd);
}
private string serializeMakeFlagsImpl(CliConfig config, int readFd, int writeFd)
{
string flags;
@@ -59,5 +74,14 @@ string serializeMakeFlags(CliConfig config)
flags ~= " -P";
if (config.debugMode)
flags ~= " -d";
// Jobserver pipe file descriptors for recursive make coordination.
// Only included when the pool has created a jobserver pipe.
if (readFd > 0 && writeFd > 0)
{
import std.conv : to;
flags ~= " --jobserver-auth=" ~ readFd.to!string ~ "," ~ writeFd.to!string;
}
return flags.strip;
}
source/antelope/shell/process.d
+126 -10
View File
@@ -4,20 +4,118 @@
/// Commands are run via `/bin/sh -c` on POSIX systems.
module antelope.shell.process;
import std.process : spawnProcess, wait;
import std.process : spawnProcess, spawnShell, wait, Pid, Pipe, pipeShell,
Redirect, ProcessPipes, Config;
import std.string : indexOf;
import std.stdio : File;
/// Run a command via shell and return its exit code.
/// Handle to a running piped subprocess.
///
/// Created by `runProcessPiped()`, this lets the caller read
/// stdout/stderr asynchronously and wait for completion.
struct ProcessHandle
{
Pid pid; /// Process ID
File stdoutPipe; /// File for reading process stdout
File stderrPipe; /// File for reading process stderr
string command; /// The command that was executed (for error messages)
/// Wait for the process to finish and return its exit code.
/// Returns: the exit code, or -1 if waiting failed.
int waitFor()
{
try
{
return wait(pid);
}
catch (Exception)
{
return -1;
}
}
/// Close the stdout and stderr pipes.
/// Must be called after reading all output to avoid fd leaks.
void closePipes()
{
try { stdoutPipe.close(); } catch (Exception) {}
try { stderrPipe.close(); } catch (Exception) {}
}
}
/// Run a command via shell with piped stdout and stderr.
///
/// Unlike `runProcess()` which blocks and inherits parent fds,
/// this variant captures stdout and stderr into pipes so output
/// can be buffered and printed atomically by the caller.
///
/// Params:
/// command = The raw shell command to execute
/// environment = Optional KEY=VALUE pairs for the process environment
///
/// Returns: A ProcessHandle that can be used to read output and wait.
///
/// Throws: Exception if process spawning fails.
ProcessHandle runProcessPiped(string command, string[] environment = [])
{
// Determine shell — respect SHELL variable, default to /bin/sh
string shell = "/bin/sh";
foreach (env; environment)
{
if (env.length > 6 && env[0 .. 6] == "SHELL=")
{
shell = env[6 .. $];
break;
}
}
if (environment.length > 0)
{
string[string] envMap;
foreach (env; environment)
{
auto idx = env.indexOf('=');
if (idx != -1)
envMap[env[0 .. idx]] = env[idx + 1 .. $];
}
auto pipes = pipeShell(command,
Redirect.stdout | Redirect.stderr,
envMap, Config.none, null, shell);
ProcessHandle h;
h.pid = pipes.pid;
h.stdoutPipe = pipes.stdout;
h.stderrPipe = pipes.stderr;
h.command = command;
return h;
}
else
{
auto pipes = pipeShell(command,
Redirect.stdout | Redirect.stderr,
null, Config.none, null, shell);
ProcessHandle h;
h.pid = pipes.pid;
h.stdoutPipe = pipes.stdout;
h.stderrPipe = pipes.stderr;
h.command = command;
return h;
}
}
/// Run a command via shell and return its exit code (blocking, no capture).
///
/// This is the original synchronous variant — used for simple cases
/// and single-job builds where output capture is unnecessary.
///
/// If `command` is empty, returns 0 immediately without spawning.
/// The shell used is determined by the SHELL environment variable,
/// defaulting to /bin/sh if not set.
/// If `environment` is non-empty, it is parsed as KEY=VALUE pairs and
/// passed as the process environment; otherwise the parent
/// process environment is inherited.
/// Stdout and stderr are inherited from the parent (not captured here).
///
/// Returns: the exit code of the command, or -1 if spawning failed.
int runProcess(string command, string[] environment)
int runProcess(string command, string[] environment = [])
{
if (command.length == 0)
return 0;
@@ -26,9 +124,9 @@ int runProcess(string command, string[] environment)
string shell = "/bin/sh";
foreach (env; environment)
{
if (env.length > 6 && env[0..6] == "SHELL=")
if (env.length > 6 && env[0 .. 6] == "SHELL=")
{
shell = env[6..$];
shell = env[6 .. $];
break;
}
}
@@ -54,7 +152,7 @@ int runProcess(string command, string[] environment)
return wait(pid);
}
}
catch (Exception e)
catch (Exception)
{
return -1;
}
@@ -65,3 +163,21 @@ unittest
assert(runProcess("echo hello", []) == 0);
assert(runProcess("exit 42", []) == 42);
}
/// Verify that `runProcessPiped` returns the same exit code as `runProcess`.
unittest
{
auto h = runProcessPiped("echo hello", []);
int code = h.waitFor();
assert(code == 0);
h.closePipes();
}
/// Test piped stderr capture via non-zero exit.
unittest
{
auto h = runProcessPiped("exit 7", []);
int code = h.waitFor();
assert(code == 7);
h.closePipes();
}