From 969e36c5ff50a0d2904f1f2961e445b703efa81a Mon Sep 17 00:00:00 2001 From: HuntedByTheIRS Date: Tue, 28 Jul 2026 22:17:49 -0400 Subject: [PATCH] Concurrency --- CHANGELOG.md | 28 +- CONTRIBUTING.md | 244 +++++- LICENSE | 26 +- Makefile | 1 + README.md | 146 +++- antefile | 2 +- dub.json | 3 +- examples/advanced/Makefile | 71 ++ examples/basic/Makefile | 15 + examples/basic/antefile | 20 + examples/basic/hello.c | 7 + examples/parallel/Makefile | 17 + examples/special-targets/Makefile | 91 +++ examples/special-targets/antefile | 88 +++ source/antelope/build/dependency.d | 171 ++++- source/antelope/build/executor.d | 253 ++++++- source/antelope/build/graph.d | 172 +++++ source/antelope/build/output.d | 154 ++++ source/antelope/build/pool.d | 922 +++++++++++++++++++++++ source/antelope/build/target.d | 43 ++ source/antelope/cli/subcommands.d | 184 +++-- source/antelope/compatibility/parallel.d | 34 +- source/antelope/compatibility/submake.d | 36 +- source/antelope/shell/process.d | 136 +++- tests/build/graph_test.d | 35 + tests/compatibility/gnu_make_test.d | 44 ++ tests/evaluator/expansion_test.d | 37 + tests/parser/lexer_test.d | 67 ++ tests/parser/parser_test.d | 30 + 29 files changed, 2966 insertions(+), 111 deletions(-) create mode 120000 Makefile create mode 100644 examples/advanced/Makefile create mode 100644 examples/basic/Makefile create mode 100644 examples/basic/antefile create mode 100644 examples/basic/hello.c create mode 100644 examples/parallel/Makefile create mode 100644 examples/special-targets/Makefile create mode 100644 examples/special-targets/antefile create mode 100644 source/antelope/build/output.d create mode 100644 source/antelope/build/pool.d create mode 100644 tests/build/graph_test.d create mode 100644 tests/compatibility/gnu_make_test.d create mode 100644 tests/evaluator/expansion_test.d create mode 100644 tests/parser/lexer_test.d create mode 100644 tests/parser/parser_test.d diff --git a/CHANGELOG.md b/CHANGELOG.md index 83469cf..ab8204d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +All notable changes to Antelope are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + ## [Unreleased] -- Initial project scaffolding. +### Added +- Project scaffolding: full module layout with type definitions and structural + skeleton across all layers (CLI, parser, evaluator, build, shell, filesystem, + compatibility, diagnostics). +- CLI argument parsing with subcommand dispatch (`build`, `hunt`, `configure`). +- `-gnu` flag for GNU Make compatibility mode (Makefile reading, full GNU Make + semantics, implicit rules, automatic variables, VPATH). +- AST node, token, and error type definitions. +- Lexer, parser, and evaluator stub modules with correct signatures. +- Build engine skeleton: dependency graph, target metadata, scheduler, executor. +- Shell interface stubs: command parsing, environment management, subprocess + execution. +- Filesystem stubs: globbing, timestamp comparison, path resolution. +- Compatibility layer: 13 modules covering the full GNU Make feature surface + (automatic variables, implicit rules, pattern rules, VPATH, target-specific + variables, order-only prerequisites, secondary expansion, include handling, + submake protocol, parallel execution, quirks, POSIX conformance). +- Diagnostics framework: structured errors, warnings, and log-level output. +- Documentation: architecture overview, GNU Make compatibility reference. +- Test fixtures for integration testing (simple builds, variables, conditionals, + include directives). +- Integration test verifying CLI argument parsing. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6fc9227..b8b640c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,243 @@ -# Contributing +# Contributing to Antelope -See the [architecture docs](docs/architecture.md) to understand the design. +## Getting Started + +### Prerequisites + +- A D compiler: [DMD](https://dlang.org/download.html) (fast compile, for + development) or [LDC](https://github.com/ldc-developers/ldc) (fast runtime, + for release builds). +- [Dub](https://code.dlang.org/getting_started) — the D package manager + (bundled with DMD). + +### Setup + +```sh +git clone https://git.spectoria.dev/specter/antelope.git +cd antelope +dub build # Verify the project builds +dub test # Run the test suite +``` + +### Build Commands + +| Command | Purpose | +|---------|---------| +| `dub build` | Build with DMD (fast compile, development) | +| `dub build --compiler=ldc2` | Build with LDC (optimized, release) | +| `dub run` | Build and run | +| `dub test` | Run all unit tests | +| `dub test --compiler=ldc2` | Run tests with LDC | + +## Project Structure + +``` +antelope/ +├── dub.json Package descriptor +├── source/antelope/ Source code +│ ├── app.d Entry point +│ ├── cli/ Command-line interface +│ ├── parser/ Lexer, parser, AST +│ ├── evaluator/ AST evaluation, expansion +│ ├── build/ Dependency graph, scheduling, execution +│ ├── shell/ Command parsing, environment, subprocess +│ ├── filesystem/ File I/O, glob, timestamps, paths +│ ├── compatibility/ GNU Make emulation (13 modules) +│ └── diagnostics/ Errors, warnings, structured output +├── tests/ Test suites (mirrors source layout) +├── examples/ Example build files +└── docs/ Design documentation +``` + +Before diving in, read: +- [docs/architecture.md](docs/architecture.md) — system design and key decisions +- [docs/compatibility.md](docs/compatibility.md) — GNU Make feature coverage + +## Coding Conventions + +### D Language Style + +**Naming:** +- Types (structs, enums, classes): `PascalCase` (`AstNode`, `TargetKind`) +- Functions: `camelCase` (`parseCommand`, `resolveDependencies`) +- Variables: `camelCase` (`lexerState`, `targetName`) +- Enum members: `snake_case` (`notparallel`, `fileNotFound`) +- Constants: `camelCase` with `static immutable` or `enum` +- Modules: `snake_case` matching directory structure (`antelope.parser.ast`) + +**Documentation:** +- All public declarations get `///` doc comments (ddoc format) +- Module-level doc comment at the top of each file +- Struct fields get per-field `///` comments +- Complex algorithms get an explanation or example + +**Imports:** +- Explicit selective imports preferred over wildcard imports +- Group: standard library imports first, then project imports +- Within project: order by module depth (shallowest first) + +**Error handling:** +- Return error structs over throwing exceptions for expected failures +- Use `ErrorKind` enum for categorized errors +- Exceptions only for truly exceptional conditions (OOM, unexpected state) +- No `assert(false)` — use structured error reporting + +**Memory:** +- Use D's GC freely (not a systems-level binary) +- Avoid manual memory management unless in hot paths +- Prefer slices (`[]`) over dynamic arrays for function parameters + +### Code Organization + +- One logical concept per module +- Maximum ~300 lines per module (split if longer) +- Module name matches file name exactly +- Public API at the top of the file, implementation details below + +### Compat-Reader Pattern + +Modules that have both "clean" and "compat" behavior gate the compat path +behind the `-gnu` flag: + +```d +/// Pure implementation (Antelope-native behavior) +string expand(string input) { ... } + +/// Compat path (call for GNU Make emulation, enabled via -gnu) +string expandCompat(string input, GnuMakeCompat compat) +{ + if (compat.enableGnuBuiltins) + return expandGnuStyle(input); + return expand(input); +} +``` + +This keeps the clean implementation readable while providing compatibility +as a layer on top. + +### Module Template + +```d +/// One-line summary of what this module does. +/// +/// Extended description with details, edge cases, and +/// references to related modules or GNU Make behavior. +module antelope.layer.module; // Path from source/ + +import std.algorithm; +import antelope.diagnostics.errors; + +// --- Types --- + +/// Brief doc for the primary type. +struct MyType +{ + string name; + size_t count; +} + +// --- Public API --- + +/// Calculate something from input. +/// Returns: description of return value. +ReturnType doThing(InputType input) +{ + return ReturnType(); +} + +// --- Implementation --- + +private void helperFunction() +{ +} +``` + +## Testing + +### Test Layout + +``` +tests/ +├── parser/ Lexer and parser unit tests +├── evaluator/ Expansion and conditional tests +├── build/ Graph, scheduling, and execution tests +├── compatibility/ GNU Make behavior conformance tests +└── integration/ End-to-end Makefile execution tests +``` + +### Testing Philosophy + +- **Unit tests** for each module in isolation (mock dependencies where needed) +- **Snapshot tests** for parser output (parse Makefile → compare AST) +- **Compat tests** — run real GNU Make on a test Makefile, then run Antelope, + and compare outputs (stdout, exit code, files produced) +- **Regression tests** for every quirk and bug fix + +### Running Tests + +```sh +dub test # All tests +dub test --compiler=ldc2 # All tests with LDC +``` + +Tests are `unittest` blocks within modules or in separate test files. +`dub test` discovers and runs all `unittest` blocks. + +### Test Structure Pattern + +```d +// tests/parser/lexer_test.d +unittest +{ + auto lexer = Lexer("target: prereq\n\trecipe line"); + assert(lexer.nextToken().type == TokenType.identifier); + assert(lexer.nextToken().type == TokenType.colon); + // ... +} +``` + +## Design Principles + +### Compatibility Always Comes First + +This is the #1 design constraint. Everything yields to compatibility. +Antelope must flawlessly run real-world Makefiles, especially those +generated by GNU Autotools. If a Makefile works with GNU Make, it must +work with Antelope. + +### Explicitness over Implicit + +Antelope's native mode is explicit by design. Implicit behaviors (pattern +rules, automatic variables, derived prerequisites) are only enabled when +explicitly requested — either via `-gnu` for full GNU compat, or via +per-feature opt-ins. This keeps builds predictable and debuggable. + +### Correctness First + +Dependency resolution must be DAG-accurate with no spurious rebuilds. +Timestamp comparison must be monotonic and race-condition safe. + +### Structs over Classes + +Prefer D structs over classes for data types — value semantics, stack +allocation, better cache locality. Exceptions: subsystems that genuinely +need polymorphism or reference semantics (e.g., jobserver coordination). + +### The `-gnu` Flag as Mode Switch + +The `-gnu` flag is the architectural cornerstone. It is NOT just a feature +flag — it changes behavior at every layer of the system, from which build +file is read to which parser rules, implicit rules, and variables are +active. The flag is checked at the top level and threaded through as +configuration, not as a global variable. + +## Commit Guidelines + +- Write commits in imperative mood: "Add lexer tokenization for recipe lines" +- Keep commits atomic — one logical change per commit +- Reference relevant issues or design docs where applicable +- No AI-generated commit messages — write meaningful, human-authored messages + +## Questions? + +Open an issue or start a discussion on the project repository. diff --git a/LICENSE b/LICENSE index 977b8df..425b9b7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,29 @@ BSD 3-Clause License Copyright (c) 2026, Antelope Contributors - All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 120000 index 0000000..47a868c --- /dev/null +++ b/Makefile @@ -0,0 +1 @@ +antefile \ No newline at end of file diff --git a/README.md b/README.md index 9af24e0..2245f52 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,147 @@ # Antelope -A full replacement for GNU Make, with compatibility in the 99th percentile. -Written in D. +**A ground-up replacement for GNU Make and GNU Autotools, written in D.** -## Status +Antelope offers direct, high-fidelity compatibility with existing Makefiles +while providing a modern, fast, and correct build system. It targets +99th-percentile GNU Make compatibility — every quirk, feature, and bug that +real-world Makefiles depend on is catalogued and emulated. -Early development. +## Why Antelope? + +- **Compatibility first** — runs real-world Makefiles, especially those generated + by GNU Autotools. This is the #1 design constraint. +- **Single binary** — no m4, no shell scripts, no generated intermediates. + One `antelope` binary replaces `make`, `autoconf`, `automake`, and `libtool`. +- **Correctness** — DAG-accurate dependency resolution with no spurious rebuilds. + Timestamp comparison is monotonic and race-condition safe. +- **Performance** — compiled D binary with parallel scheduling, lazy evaluation, + and microsecond startup times. +- **Two modes, cleanly separated** — native mode is explicit by design; GNU + compatibility is activated via a single `-gnu` flag. + +## Quick Start + +### Building + +```sh +# Development build (fast compile, slower runtime) +dub build + +# Release build (slower compile, fast runtime) +dub build --compiler=ldc2 + +# Run tests +dub test + +# Run with full GNU Make compatibility +dub run -- -gnu + +# Run a specific Makefile +dub run -- -gnu -f MyMakefile target1 target2 +``` + +### CLI Usage + +``` +antelope --[flags] +``` + +| Command | Description | +|---------|-------------| +| `antelope` | Run the build (default subcommand) | +| `antelope build` | Explicit build invocation | +| `antelope hunt` | Intelligent Makefile → Antefile converter (late-stage) | +| `antelope configure` | Autotools configure.ac replacement (future) | + +| Flag | Description | +|------|-------------| +| `-gnu` | Enable full GNU Make compatibility mode | +| `-f ` | Specify a build file | +| `-j ` | Run N jobs in parallel | +| `-n` | Dry run (print commands without executing) | +| `-C ` | Change to directory before reading build files | +| `-d` | Debug output | +| `-P` | POSIX conformance mode | + +## Two Modes + +### Native Mode (default) + +Antelope's native mode is explicit by design — no implicit rules, no automatic +variables, no magic. The build file is an `antefile` or `antelope` file. + +```make +# antefile — explicit, predictable, debuggable +hello: hello.o + gcc -o hello hello.o + +hello.o: hello.c + gcc -c hello.c +``` + +### GNU Mode (`-gnu`) + +Pass `-gnu` and Antelope becomes GNU Make — `Makefile` / `makefile` reading, +implicit rules, automatic variables (`$@`, `$<`, `$^`), VPATH, pattern rules, +conditionals, and the entire GNU Make compatibility surface. + +```sh +antelope -gnu -j8 # Run Makefile with 8 parallel jobs +``` + +## Architecture + +Antelope is organized into clean layers: + +| Layer | Purpose | +|-------|---------| +| **CLI** | Subcommand dispatch, argument parsing, help output | +| **Parser** | Lexer → recursive-descent parser → AST | +| **Evaluator** | AST walking, variable expansion, conditionals, built-in functions | +| **Build** | Dependency graph, topological scheduling, recipe execution | +| **Shell** | Command parsing, environment management, subprocess execution | +| **Filesystem** | Globbing, timestamp comparison, path resolution | +| **Compatibility** | GNU Make feature emulation (13 modules) | +| **Diagnostics** | Structured errors, warnings, log-level output | + +See [docs/architecture.md](docs/architecture.md) for the full design. + +## GNU Make Compatibility + +Antelope targets GNU Make versions 3.81 through 4.4 with feature-complete +emulation of: + +- Automatic variables (`$@`, `$<`, `$^`, `$*`, `$?`, `$%`, `$+`, `$|`) +- ~30 built-in implicit rules (C, C++, Fortran, lex, yacc, archive management) +- Pattern rules (`%.o: %.c`) and suffix rules (`.c.o:`) +- VPATH / vpath directory search +- Target-specific and pattern-specific variable assignments +- Order-only prerequisites (`|` separator) +- Secondary expansion (`.SECONDEXPANSION`) +- Include directives with makefile remaking and restart +- Recursive `$(MAKE)` with MAKEFLAGS propagation and jobserver protocol +- Parallel execution (`-jN`, `.NOTPARALLEL`, `.WAIT`, `.JOBS`) +- Conditionals (`ifeq`/`ifneq`/`ifdef`/`ifndef`) +- Functions (`$(shell ...)`, `$(wildcard ...)`, `$(patsubst ...)`, etc.) +- All documented behavioral quirks and edge cases + +See [docs/compatibility.md](docs/compatibility.md) for the full compatibility matrix. + +## Project Status + +**Phase:** Early scaffolding. Stub modules with type definitions and structural +skeleton. Core implementation (lexer, parser, evaluator, build engine) is the +active development focus. + +## Documentation + +- [Architecture](docs/architecture.md) — system design and key decisions +- [Compatibility](docs/compatibility.md) — GNU Make feature coverage +- [Manual](docs/manual.md) — user guide and reference +- [Contributing](CONTRIBUTING.md) — how to contribute + +## License + +BSD 3-Clause. See [LICENSE](LICENSE) for the full text. diff --git a/antefile b/antefile index 89995ed..ddf8c28 100644 --- a/antefile +++ b/antefile @@ -29,7 +29,7 @@ test: dub test clean: - rm -f $(OUT) + rm -f $(OUT) libantelope.a dist-clean: clean rm -f antelope-*.tar.zst diff --git a/dub.json b/dub.json index 9a7e92f..4b2d4a1 100644 --- a/dub.json +++ b/dub.json @@ -5,5 +5,6 @@ "Specter" ], "copyright": "Copyright (c) 2026 Antelope Contributors", - "name": "antelope" + "name": "antelope", + "targetType": "executable" } \ No newline at end of file diff --git a/examples/advanced/Makefile b/examples/advanced/Makefile new file mode 100644 index 0000000..5a1c06c --- /dev/null +++ b/examples/advanced/Makefile @@ -0,0 +1,71 @@ +# Advanced Makefile showcasing pattern rules, VPATH, conditionals, +# functions, automatic variables, and target-specific variables. +# +# Usage: antelope -gnu (builds "all") +# antelope -gnu DEBUG=1 (debug build) +# antelope -gnu check (run the test target) +# antelope -gnu clean (clean build artifacts) + +# ---- Project Configuration ---- + +PROJECT = myapp +SRCDIR = src +BUILDDIR = build +VPATH = $(SRCDIR) + +# ---- Conditional Configuration ---- + +ifeq ($(DEBUG),1) + CFLAGS = -g -O0 -DDEBUG + BUILDDIR = build/debug +else + CFLAGS = -O2 -DNDEBUG + BUILDDIR = build/release +endif + +# ---- Auto-discovered sources ---- + +SRCS := $(notdir $(wildcard $(SRCDIR)/*.c)) +OBJS := $(patsubst %.c,$(BUILDDIR)/%.o,$(SRCS)) + +# ---- Targets ---- + +.PHONY: all clean check + +all: $(BUILDDIR)/$(PROJECT) + +# Output directory creation (order-only prerequisite — timestamp doesn't +# trigger rebuild, but the directory must exist before the recipe runs). +$(BUILDDIR)/$(PROJECT): $(OBJS) | $(BUILDDIR) + $(CC) $(CFLAGS) -o $@ $^ + +# Pattern rule: build .o from .c, placing output in BUILDDIR +$(BUILDDIR)/%.o: %.c + $(CC) $(CFLAGS) -c -o $@ $< + +# Create the output directory +$(BUILDDIR): + mkdir -p $@ + +# ---- Per-target variables (release target gets extra optimization) ---- + +$(BUILDDIR)/release/$(PROJECT): CFLAGS += -flto + +# ---- Utility targets ---- + +check: $(BUILDDIR)/$(PROJECT) + @echo "Running tests..." + ./$(BUILDDIR)/$(PROJECT) --test + @echo "All tests passed." + +clean: + rm -rf $(BUILDDIR) + +# ---- Informational ---- + +info: + $(info Project: $(PROJECT)) + $(info Sources: $(SRCS)) + $(info Objects: $(OBJS)) + $(info CFLAGS: $(CFLAGS)) + $(info Build dir: $(BUILDDIR)) diff --git a/examples/basic/Makefile b/examples/basic/Makefile new file mode 100644 index 0000000..3318768 --- /dev/null +++ b/examples/basic/Makefile @@ -0,0 +1,15 @@ +# Basic GNU Make build — uses implicit rules +# +# Usage: antelope -gnu (builds "all" using implicit rules) +# antelope -gnu clean (cleans build artifacts) +# +# In GNU mode, the %.o: %.c implicit rule handles hello.c → hello.o for free. + +all: hello + +hello: hello.o + +.PHONY: clean + +clean: + rm -f hello hello.o diff --git a/examples/basic/antefile b/examples/basic/antefile new file mode 100644 index 0000000..309268f --- /dev/null +++ b/examples/basic/antefile @@ -0,0 +1,20 @@ +# Basic Antelope build — native mode (explicit, no magic) +# +# Usage: antelope (builds "all") +# antelope clean (cleans build artifacts) + +CC = gcc +CFLAGS = -Wall -O2 + +all: hello + +hello: hello.o + $(CC) $(CFLAGS) -o hello hello.o + +hello.o: hello.c + $(CC) $(CFLAGS) -c hello.c + +.PHONY: clean + +clean: + rm -f hello hello.o diff --git a/examples/basic/hello.c b/examples/basic/hello.c new file mode 100644 index 0000000..3b5d288 --- /dev/null +++ b/examples/basic/hello.c @@ -0,0 +1,7 @@ +#include + +int main(void) +{ + printf("Hello from Antelope!\n"); + return 0; +} diff --git a/examples/parallel/Makefile b/examples/parallel/Makefile new file mode 100644 index 0000000..03ee52a --- /dev/null +++ b/examples/parallel/Makefile @@ -0,0 +1,17 @@ +# Parallel build example — multiple independent sub-projects +# +# Usage: antelope -gnu -j4 (build all modules in parallel with 4 jobs) +# antelope -gnu -j0 (build with unlimited parallelism) +# antelope -gnu clean (clean all modules) + +SUBDIRS = module_a module_b module_c + +.PHONY: all clean $(SUBDIRS) + +all clean: $(SUBDIRS) + +$(SUBDIRS): + $(MAKE) -C $@ $(MAKECMDGOALS) + +# .WAIT ensures module_c starts only after module_a and module_b finish. +module_c: module_a module_b .WAIT diff --git a/examples/special-targets/Makefile b/examples/special-targets/Makefile new file mode 100644 index 0000000..1ea6c91 --- /dev/null +++ b/examples/special-targets/Makefile @@ -0,0 +1,91 @@ +# .WAIT and .JOBS examples — GNU Make mode (-gnu) +# +# Usage: +# antelope -gnu -j4 Build all targets with 4 parallel jobs +# antelope -gnu clean Clean build artifacts +# +# .WAIT splits prerequisites into sequential groups: +# target: group1 .WAIT group2 +# → group1 completes first, then group2 starts +# +# .JOBS limits concurrency for named targets (GNU Make 4.4+): +# .JOBS: 2 heavy_a heavy_b +# → at most 2 of {heavy_a, heavy_b} run concurrently + +# ── Example 1: .WAIT ordering barrier ────────────────────────────── +# +# In a multi-stage build, .WAIT ensures earlier stages complete +# before later stages begin. Targets on the same side of .WAIT +# can run in parallel. + +all: stage1 stage2 stage3 + +# Stage 1: independent tasks — all run in parallel +stage1: task_a task_b + @echo "[stage1] all prerequisite tasks complete" + +task_a: + @echo "[task_a] running..." + @sleep 0.2 + @echo "[task_a] done" + +task_b: + @echo "[task_b] running..." + @sleep 0.2 + @echo "[task_b] done" + +# Stage 2: must wait for stage1, then runs two tasks in parallel +stage2: stage1 .WAIT task_c task_d + @echo "[stage2] all prerequisite tasks complete" + +task_c: + @echo "[task_c] running..." + @sleep 0.3 + @echo "[task_c] done" + +task_d: + @echo "[task_d] running..." + @sleep 0.3 + @echo "[task_d] done" + +# Stage 3: must wait for stage2 +stage3: stage2 .WAIT + @echo "[stage3] final stage complete" + +# ── Example 2: .JOBS resource throttle ───────────────────────────── +# +# When some targets are resource-heavy (CPU, memory, I/O), .JOBS +# caps their concurrency while allowing lighter targets to use +# the full global -j limit. + +.JOBS: 2 big_compile_a big_compile_b big_compile_c + +all_heavy: big_compile_a big_compile_b big_compile_c small_task + @echo "All done" + +big_compile_a: + @echo "[big_compile_a] compiling..." + @sleep 0.4 + @echo "[big_compile_a] done" + +big_compile_b: + @echo "[big_compile_b] compiling..." + @sleep 0.4 + @echo "[big_compile_b] done" + +big_compile_c: + @echo "[big_compile_c] compiling..." + @sleep 0.4 + @echo "[big_compile_c] done" + +small_task: + @echo "[small_task] done instantly" + +# ── Housekeeping ──────────────────────────────────────────────────── + +.PHONY: all stage1 stage2 stage3 task_a task_b task_c task_d +.PHONY: all_heavy big_compile_a big_compile_b big_compile_c small_task +.PHONY: clean + +clean: + @echo "Cleaning..." diff --git a/examples/special-targets/antefile b/examples/special-targets/antefile new file mode 100644 index 0000000..0dabc31 --- /dev/null +++ b/examples/special-targets/antefile @@ -0,0 +1,88 @@ +# .WAIT and .JOBS examples — native mode +# +# Usage: +# antelope Build all targets +# antelope -j4 Build with 4 parallel jobs +# antelope clean Clean build artifacts +# +# .WAIT splits prerequisites into sequential groups: +# target: group1 .WAIT group2 +# → group1 completes first, then group2 starts +# +# .JOBS limits concurrency for named targets: +# .JOBS: N target1 target2 ... +# → target1 and target2 run at most N jobs concurrently + +# ── Example 1: .WAIT barrier ─────────────────────────────────────── +# +# pipeline: fetch .WAIT build .WAIT test +# → fetch runs, then build after fetch, then test after build +# Within each group, targets can run in parallel. + +pipeline: fetch build test + @echo "Pipeline complete" + +fetch: + @echo "Fetching dependencies..." + @sleep 0.2 + @echo "Fetch done" + +build: compile_a compile_b + @echo "Build complete" + +compile_a: + @echo "Compiling module A..." + @sleep 0.3 + @echo "Module A done" + +compile_b: + @echo "Compiling module B..." + @sleep 0.3 + @echo "Module B done" + +# .WAIT ensures test runs only after build completes +test: build .WAIT + @echo "Running tests..." + @sleep 0.1 + @echo "All tests passed" + +# ── Example 2: .JOBS throttle ────────────────────────────────────── +# +# .JOBS: 2 heavy_a heavy_b heavy_c +# → Only 2 of {heavy_a, heavy_b, heavy_c} run concurrently +# Light targets (light_x, light_y) use the global -j limit. + +.JOBS: 2 heavy_a heavy_b heavy_c + +heavy_jobs: heavy_a heavy_b heavy_c light_x light_y + @echo "All heavy and light jobs done" + +heavy_a: + @echo "[heavy_a] starting..." + @sleep 0.5 + @echo "[heavy_a] done" + +heavy_b: + @echo "[heavy_b] starting..." + @sleep 0.5 + @echo "[heavy_b] done" + +heavy_c: + @echo "[heavy_c] starting..." + @sleep 0.5 + @echo "[heavy_c] done" + +light_x: + @echo "[light_x] done instantly" + +light_y: + @echo "[light_y] done instantly" + +# ── Housekeeping ──────────────────────────────────────────────────── + +.PHONY: pipeline fetch build compile_a compile_b test +.PHONY: heavy_jobs heavy_a heavy_b heavy_c light_x light_y +.PHONY: clean + +clean: + @echo "Cleaning..." diff --git a/source/antelope/build/dependency.d b/source/antelope/build/dependency.d index 9b1477f..6546bf9 100644 --- a/source/antelope/build/dependency.d +++ b/source/antelope/build/dependency.d @@ -1,9 +1,8 @@ /// Dependency resolution and ordering logic. /// /// Uses Kahn's algorithm (BFS-based topological sort) to produce ordered -/// build batches. Each batch contains targets that can be built in parallel. -/// The first batch contains leaf targets (no unresolved prereqs in the -/// graph), and the last batch contains the requested root target. +/// build batches, plus critical-path weight computation for load-aware +/// scheduling. module antelope.build.dependency; import antelope.build.graph; @@ -152,6 +151,124 @@ Target[][] resolveDependencies(DependencyGraph graph, string target) return batches; } +/// Compute critical-path weights for every target reachable from `root`. +/// +/// The critical-path weight of a target is: +/// weight = recipe.length + max(weight of each successor) +/// +/// A "successor" is any target that depends on this target (i.e., a +/// target listing this one as a prerequisite). This is the reverse +/// of the usual dependency direction — we compute from the root +/// backward to the leaves. +/// +/// Leaf nodes (targets with no in-graph dependents) have weight = recipe.length. +/// Root nodes accumulate the full chain of work beneath them. +/// +/// The resulting weights are written directly into each `Target.criticalWeight` +/// field. The caller sorts the ready queue by descending `criticalWeight` +/// to prioritize targets on the critical path. +/// +/// Params: +/// graph = The dependency graph with reverse edges already populated +/// via `DependencyGraph.buildReverseEdges()`. +/// root = The root target name to start the weight computation from. +void computeCriticalWeights(ref DependencyGraph graph, string root) +{ + import std.algorithm : max; + + // Only consider targets reachable from the root. + bool[string] reachable; + { + string[] stack = [root]; + while (stack.length > 0) + { + string current = stack[$ - 1]; + stack = stack[0 .. $ - 1]; + if (current in reachable) + continue; + reachable[current] = true; + auto tp = graph.findTarget(current); + if (tp is null) + continue; + foreach (dep; tp.prerequisites ~ tp.orderOnlyPrereqs) + if (graph.hasTarget(dep) && dep !in reachable) + stack ~= dep; + } + } + + // Build a dependency map: node → all in-graph prereqs + string[][string] prereqMap; + foreach (ref t; graph.targets) + { + if (t.name !in reachable) + continue; + foreach (p; t.prerequisites ~ t.orderOnlyPrereqs) + if (p in reachable && graph.hasTarget(p)) + prereqMap[t.name] ~= p; + } + + // Kahn-style topological order from leaves (in-degree 0) to root. + size_t[string] inDegree; + string[][string] dependentsMap; // prereq → dependents + + foreach (name; reachable.keys) + inDegree[name] = 0; + + foreach (name, prereqs; prereqMap) + { + inDegree[name] = prereqs.length; + foreach (p; prereqs) + dependentsMap[p] ~= name; + } + + // Process in topological order: all of a node's prereqs are + // processed before the node itself, so their weights are finalised. + string[] queue; + foreach (name; reachable.keys) + if (inDegree[name] == 0) + queue ~= name; + + string[] order; + while (queue.length > 0) + { + string current = queue[$ - 1]; + queue = queue[0 .. $ - 1]; + order ~= current; + + auto deps = current in dependentsMap; + if (deps is null) + continue; + foreach (dep; *deps) + { + inDegree[dep]--; + if (inDegree[dep] == 0) + queue ~= dep; + } + } + + // Now compute weights in topological order. + // order[0] = leaf, order[$-1] = root. + foreach (name; order) + { + auto tp = graph.findTarget(name); + if (tp is null) + continue; + + size_t maxPrereqWeight = 0; + auto prereqs = name in prereqMap; + if (prereqs) + { + foreach (p; *prereqs) + { + auto pp = graph.findTarget(p); + if (pp !is null) + maxPrereqWeight = max(maxPrereqWeight, pp.criticalWeight); + } + } + tp.criticalWeight = tp.recipe.length + maxPrereqWeight; + } +} + /// unittest { @@ -211,3 +328,51 @@ unittest assert(batches[1].length == 1); assert(batches[1][0].name == "program"); } + +/// Critical path weights: program(1) → main.o(1) → main.c(0) = 2 +unittest +{ + DependencyGraph g; + g.addTarget(Target("main.c", TargetKind.file, [], [])); + g.addTarget(Target("main.o", TargetKind.file, ["main.c"], + ["gcc -c main.c"])); + g.addTarget(Target("program", TargetKind.file, ["main.o"], + ["gcc -o program main.o"])); + + g.buildReverseEdges(); + computeCriticalWeights(g, "program"); + + // main.c: no recipe, no prereqs → weight 0 + auto mc = g.findTarget("main.c"); + assert(mc !is null); + assert(mc.criticalWeight == 0); + + // main.o: 1 recipe line, prereq main.c (weight 0) → weight 1 + auto mo = g.findTarget("main.o"); + assert(mo !is null); + assert(mo.criticalWeight == 1); + + // program: 1 recipe line, prereq main.o (weight 1) → weight 2 + auto prog = g.findTarget("program"); + assert(prog !is null); + assert(prog.criticalWeight == 2); +} + +/// Diamond dependency: root → a, b → leaf. Weights should reflect +/// that both branches are equal. +unittest +{ + DependencyGraph g; + g.addTarget(Target("leaf", TargetKind.file, [], ["touch leaf"])); // weight 1 + g.addTarget(Target("a", TargetKind.file, ["leaf"], ["cp leaf a"])); // weight 2 + g.addTarget(Target("b", TargetKind.file, ["leaf"], ["cp leaf b"])); // weight 2 + g.addTarget(Target("root", TargetKind.file, ["a", "b"], ["cat a b"])); // weight 3 + + g.buildReverseEdges(); + computeCriticalWeights(g, "root"); + + assert(g.findTarget("leaf").criticalWeight == 1); + assert(g.findTarget("a").criticalWeight == 2); + assert(g.findTarget("b").criticalWeight == 2); + assert(g.findTarget("root").criticalWeight == 3); +} diff --git a/source/antelope/build/executor.d b/source/antelope/build/executor.d index 17245d0..1e528e5 100644 --- a/source/antelope/build/executor.d +++ b/source/antelope/build/executor.d @@ -1,17 +1,35 @@ /// Command execution engine — runs recipe lines and reports results. +/// +/// Now supports parallel execution: `executeTarget()` spawns processes +/// with piped stdout/stderr for output buffering, while `execute()` +/// remains available for simple synchronous use. module antelope.build.executor; import antelope.shell.process; +import antelope.build.target; +import antelope.build.output; +import antelope.diagnostics.output; /// Result of executing a single recipe line. struct ExecResult { bool success; /// True if the command succeeded or ignoreErrors was set - string output; /// (reserved for future output capture) + string output; /// Captured stdout + stderr from the process int exitCode; /// Exit code from the process } -/// Execute a command string and return the result. +/// Result of executing all recipe lines for a single target. +struct JobResult +{ + string targetName; /// Name of the target that was built + bool success; /// True if all recipe lines succeeded + int exitCode; /// Last non-zero exit code (0 if all succeeded) + string[] stdoutLines; /// Captured stdout, one entry per recipe line + string[] stderrLines; /// Captured stderr, one entry per recipe line + bool hadEcho; /// True if any non-@ recipe line was executed +} + +/// Execute a command string and return the result (synchronous, no capture). /// /// Handles GNU Make recipe prefix characters (@, -, +), then passes the /// remaining line directly to /bin/sh. Tokenization is deliberately @@ -60,3 +78,234 @@ ExecResult execute(string command, string[] environment = []) bool ok = (code == 0 || ignoreErrors); return ExecResult(ok, "", code); } + +/// Execute all recipe lines for a single target, capturing output. +/// +/// Each recipe line is expanded (via the caller-supplied expander), +/// stripped of prefix characters, printed according to echo rules, +/// and executed via a piped subprocess. Output is buffered into the +/// supplied `OutputManager`. +/// +/// The `expander` delegate is called to perform variable expansion +/// (e.g., $(CC), $@, $<) at execution time. It receives: +/// - The raw recipe line text +/// - The target name (for $@ expansion) +/// - The target's prerequisites (for $<, $^ expansion) +/// - The target's stem (for $* expansion in pattern rules) +/// +/// Params: +/// t = The target to build +/// execEnv = Environment variables (KEY=VALUE) for the subprocess +/// expander = Delegate for variable expansion +/// output = Output buffer manager (may be null for live mode) +/// isDryRun = If true, print commands but don't execute +/// silentMode = If true, suppress echo of non-@ lines +/// +/// Returns: JobResult with success flag and captured output. +JobResult executeTarget( + Target t, + string[] execEnv, + string delegate(string, string, string[], string) expander, + OutputManager* output, + bool isDryRun = false, + bool silentMode = false) +{ + import std.string : stripLeft; + + JobResult result; + result.targetName = t.name; + result.success = true; + result.exitCode = 0; + + // Short-circuit: targets with no recipe are always "successful" + // (they exist on disk or are phony/intermediate markers). + if (t.recipe.length == 0) + return result; + + foreach (recipeLine; t.recipe) + { + // Expand variables in the recipe line. + // Automatic variables ($@, $<, $^, $*, etc.) are resolved + // against the current target context. + string expanded = expander(recipeLine, t.name, + t.prerequisites, t.stem); + + // Strip prefix characters to determine echo/error behaviour. + string trimmed = expanded.stripLeft(); + bool ignoreErrors; + bool silent = silentMode; + + if (trimmed.length > 0) + { + bool stripping = true; + while (stripping && trimmed.length > 0) + { + stripping = false; + switch (trimmed[0]) + { + case '@': + silent = true; + trimmed = trimmed[1 .. $]; + stripping = true; + break; + case '-': + ignoreErrors = true; + trimmed = trimmed[1 .. $]; + stripping = true; + break; + case '+': + trimmed = trimmed[1 .. $]; + stripping = true; + break; + default: + break; + } + } + } + + if (trimmed.length == 0) + continue; + + // Echo: print the command unless suppressed. + // GNU Make prints the expanded form. + string echoLine = expanded.stripLeft(); + bool shouldEcho = !silent && echoLine.length > 0; + + if (shouldEcho) + { + if (output) + { + output.bufferStdout(t.name, echoLine); + output.markEchoed(t.name); + result.hadEcho = true; + } + else + { + log(LogLevel.normal, echoLine); + } + } + + // Dry run: skip execution. + if (isDryRun) + { + result.stdoutLines ~= shouldEcho ? echoLine : ""; + continue; + } + + // Execute the command with piped output. + auto ph = runProcessPiped(trimmed, execEnv); + + // Read stdout and stderr from pipes. + string lineStdout; + string lineStderr; + + // Simple line-by-line reading from the pipes. + // NOTE: stdout is read first, then stderr. If the child process + // fills its stderr pipe buffer (>64KB on Linux) before stdout + // is fully drained, both sides deadlock. For typical compiler + // output this is unlikely; a future fix should drain both pipes + // concurrently via select/poll or lightweight threads. + try + { + import std.string : chomp; + + // Read stdout — ProcessHandle.stdoutPipe is a File directly. + foreach (line; ph.stdoutPipe.byLine) + { + string s = line.chomp().idup; + lineStdout ~= s ~ "\n"; + if (output) + output.bufferStdout(t.name, s); + else + log(LogLevel.normal, s); + } + + // Read stderr + foreach (line; ph.stderrPipe.byLine) + { + string s = line.chomp().idup; + lineStderr ~= s ~ "\n"; + if (output) + output.bufferStderr(t.name, s); + else + log(LogLevel.normal, s); + } + } + catch (Exception e) + { + // Log pipe errors but continue — the process exit code + // will determine success/failure. + log(LogLevel.dbg, "[" ~ t.name ~ "] pipe read error: " ~ e.msg); + } + + // Wait for the process. + int code = ph.waitFor(); + ph.closePipes(); + + // Store output + result.stdoutLines ~= lineStdout; + result.stderrLines ~= lineStderr; + + // Check result + if (code != 0 && !ignoreErrors) + { + result.success = false; + result.exitCode = code; + return result; + } + + if (code != 0) + result.exitCode = code; + } + + return result; +} + +/// +unittest +{ + // Simple synchronous execution (no expansion needed for this test). + auto r = execute("echo hello"); + assert(r.success); + assert(r.exitCode == 0); +} + +/// +unittest +{ + // Error-tolerant execution. + auto r = execute("-exit 1"); + assert(r.success); // - prefix ignores errors + assert(r.exitCode == 1); +} + +/// +unittest +{ + // Target execution with piped output. + Target t; + t.name = "test"; + t.recipe = ["echo hello world"]; + + // Identity expander (no variable substitution). + string expand(string ln, string tn, string[] pr, string st) { return ln; } + + auto result = executeTarget(t, [], &expand, null, false, false); + assert(result.success); + assert(result.exitCode == 0); +} + +/// +unittest +{ + // Target with failed recipe line. + Target t; + t.name = "failing"; + t.recipe = ["exit 3"]; + + string expand(string ln, string tn, string[] pr, string st) { return ln; } + + auto result = executeTarget(t, [], &expand, null, false, false); + assert(!result.success); + assert(result.exitCode == 3); +} diff --git a/source/antelope/build/graph.d b/source/antelope/build/graph.d index 4faed51..b9ebca5 100644 --- a/source/antelope/build/graph.d +++ b/source/antelope/build/graph.d @@ -57,6 +57,12 @@ struct DependencyGraph // .PHONY handling: mark its prerequisites as phony targets graph.handlePhony(); + // .WAIT handling: split prerequisite groups with barriers + graph.handleWait(); + + // .JOBS handling: per-target job limits (native mode) + graph.handleJobs(); + // Cycle detection: DFS with three-color marking graph.detectCycles(); @@ -120,6 +126,9 @@ private: } } + // ── Public scheduling / special-target API ────────────────────────── +public: + /// Find the ".PHONY" target (if it exists) and mark all of its /// prerequisites as phony targets. void handlePhony() @@ -137,6 +146,169 @@ private: } } + /// Process .WAIT special target: split prerequisite groups with barriers. + /// + /// Syntax: `target: group1 .WAIT group2` + /// → group1 completes first, then group2 starts. + /// Adds implicit dependencies: each group2 target depends on each group1 target. + void handleWait() + { + foreach (ref t; targets) + { + ptrdiff_t waitPos = -1; + foreach (i, p; t.prerequisites) + { + if (p == ".WAIT") + { + waitPos = cast(ptrdiff_t) i; + break; + } + } + if (waitPos < 0) + continue; + + string[] group1 = t.prerequisites[0 .. cast(size_t) waitPos]; + string[] group2 = t.prerequisites[cast(size_t) waitPos + 1 .. $]; + + // Remove .WAIT from prerequisites. + t.prerequisites = group1 ~ group2; + + // Add implicit deps: each group2 target depends on group1 targets. + foreach (g2name; group2) + { + auto g2 = findTarget(g2name); + if (g2 is null) + continue; + foreach (g1name; group1) + { + bool alreadyDepends; + foreach (p; g2.prerequisites) + if (p == g1name) { alreadyDepends = true; break; } + if (!alreadyDepends && g1name != g2.name) + g2.prerequisites ~= g1name; + } + } + } + } + + /// Process .JOBS special target (native mode). + /// + /// Syntax: `.JOBS: N target1 target2 ...` + /// Limits the named targets to at most N concurrent jobs. + void handleJobs() + { + import std.conv : to; + + Target* jobsTarget = findTarget(".JOBS"); + if (jobsTarget is null) + return; + + if (jobsTarget.prerequisites.length < 2) + return; + + size_t limit; + try + { + limit = jobsTarget.prerequisites[0].to!size_t; + } + catch (Exception) + { + return; + } + + foreach (i, name; jobsTarget.prerequisites[1 .. $]) + { + auto tp = findTarget(name); + if (tp !is null) + tp.jobLimit = limit; + } + + jobsTarget.kind = TargetKind.phony; + phonyTargets[".JOBS"] = true; + } + + /// Build reverse edges: populate `dependents` for each target. + /// For each target, scan all other targets' prerequisites and + /// add this target's name to the dependents list of each prereq. + void buildReverseEdges() + { + // Clear existing reverse edges. + foreach (ref t; targets) + t.dependents = []; + + foreach (ref t; targets) + { + foreach (prereq; t.prerequisites ~ t.orderOnlyPrereqs) + { + auto tp = findTarget(prereq); + if (tp !is null) + tp.dependents ~= t.name; + } + } + } + + /// Reset all scheduling state to defaults. + /// NOTE: jobLimit is NOT reset — it is set by handleJobs() and persists + /// across builds. + void resetSchedulingState() + { + foreach (ref t; targets) + { + t.state = BuildState.pending; + t.remainingDeps = 0; + t.criticalWeight = 0; + } + } + + /// Compute remaining in-graph prerequisite count for each target. + /// Stores the count in each target's `remainingDeps` field. + /// Only counts prerequisites that exist as graph targets. + void computeRemainingDeps() + { + foreach (ref t; targets) + { + size_t count; + foreach (prereq; t.prerequisites ~ t.orderOnlyPrereqs) + { + if (hasTarget(prereq)) + count++; + } + t.remainingDeps = count; + } + } + + /// Compute the transitive closure of a root target. + /// Returns all target names reachable from root (including root itself). + string[] transitiveClosure(string root) + { + bool[string] visited; + string[] stack = [root]; + string[] result; + + while (stack.length > 0) + { + string current = stack[$ - 1]; + stack = stack[0 .. $ - 1]; + + if (current in visited) + continue; + visited[current] = true; + result ~= current; + + auto tp = findTarget(current); + if (tp is null) + continue; + + foreach (prereq; tp.prerequisites ~ tp.orderOnlyPrereqs) + { + if (hasTarget(prereq) && prereq !in visited) + stack ~= prereq; + } + } + + return result; + } + /// Detect cycles in the dependency graph using three-color DFS. /// Stores found cycles as AntelopeError in cycleErrors. void detectCycles() diff --git a/source/antelope/build/output.d b/source/antelope/build/output.d new file mode 100644 index 0000000..116f13e --- /dev/null +++ b/source/antelope/build/output.d @@ -0,0 +1,154 @@ +/// Output buffering for parallel builds. +/// +/// When multiple targets build concurrently, interleaved stdout/stderr +/// produces unreadable output. This module buffers each job's output +/// and prints it atomically when the job completes, keeping the build +/// log coherent even under full parallelism. +/// +/// All methods are single-threaded — they're called exclusively from +/// the coordinator (main thread) after receiving results from workers. +/// Workers capture output locally and send it via JobDone messages; +/// no OutputManager access occurs in worker threads. +module antelope.build.output; + +import std.stdio : writeln, stderr; + +/// Manages per-target output buffering and atomic flush. +/// +/// Two modes are supported: +/// `buffered` — Output is captured per target and printed atomically +/// when the target finishes building. This is the default +/// for parallel builds and produces clean, readable logs. +/// `live` — Output is printed immediately as it arrives. Multiple +/// concurrent targets will interleave their output +/// (GNU Make's default behaviour with `-j`). +class OutputManager +{ + /// Whether to buffer output (true) or print live (false). + bool buffered = true; + +private: + /// Per-target output buffers, keyed by target name. + string[][string] stdoutBufs; + string[][string] stderrBufs; + + /// Map of target name → whether the target printed a non-@ line. + /// Used for GNU Make `-s` / `--silent` mode and @ prefix handling. + bool[string] hadEcho; + +public: + /// Buffer a line of stdout for a target. + void bufferStdout(string targetName, string line) + { + stdoutBufs[targetName] ~= line; + } + + /// Buffer a line of stderr for a target. + void bufferStderr(string targetName, string line) + { + stderrBufs[targetName] ~= line; + } + + /// Record that the target printed (or would print) a command line. + /// Used to suppress "nothing to be done" messages when commands were echoed. + void markEchoed(string targetName) + { + hadEcho[targetName] = true; + } + + /// Check whether the target echoed any command lines. + bool hasEchoed(string targetName) + { + return (targetName in hadEcho) !is null; + } + + /// Print a line immediately without buffering (live mode fallback). + void printLive(string line) + { + writeln(line); + } + + /// Print all buffered output for a completed target. + /// + /// Stdout lines are printed first, then stderr. + /// Called by the coordinator after a worker reports completion. + void flush(string targetName) + { + string[] stdoutLines; + string[] stderrLines; + + auto soPtr = targetName in stdoutBufs; + if (soPtr) + { + stdoutLines = *soPtr; + stdoutBufs.remove(targetName); + } + auto sePtr = targetName in stderrBufs; + if (sePtr) + { + stderrLines = *sePtr; + stderrBufs.remove(targetName); + } + + if (stdoutLines.length == 0 && stderrLines.length == 0) + return; + + foreach (line; stdoutLines) + writeln(line); + + if (stderrLines.length > 0) + { + foreach (line; stderrLines) + stderr.writeln(line); + stderr.flush(); + } + } + + /// Clear all buffers without printing (used on failure cleanup). + void clearAll() + { + stdoutBufs = null; + stderrBufs = null; + hadEcho = null; + } +} + +/// +unittest +{ + auto om = new OutputManager(); + + // Buffered mode: nothing printed during buffering + om.bufferStdout("foo.o", "cc -c foo.c"); + om.bufferStdout("foo.o", "foo.c: In function 'main':"); + om.bufferStderr("foo.o", "foo.c:5: warning: unused variable 'x'"); + + // Flush should produce all lines in order + om.flush("foo.o"); + + // After flush, buffers are empty — second flush is a no-op + om.flush("foo.o"); + + // Live mode: serialized printing + om.buffered = false; + om.printLive("live output line"); +} + +/// +unittest +{ + auto om = new OutputManager(); + + // Multiple targets interleaved buffering + om.bufferStdout("a", "building a"); + om.bufferStdout("b", "building b"); + om.bufferStdout("a", "a done"); + + om.flush("a"); + om.flush("b"); + + // echo tracking + om.markEchoed("c"); + assert(om.hasEchoed("c")); + assert(!om.hasEchoed("nonexistent")); +} diff --git a/source/antelope/build/pool.d b/source/antelope/build/pool.d new file mode 100644 index 0000000..26c6a36 --- /dev/null +++ b/source/antelope/build/pool.d @@ -0,0 +1,922 @@ +/// Parallel build worker pool with dependency-aware scheduling. +/// +/// Uses D's `std.concurrency` Actor model for worker coordination: +/// - Workers are spawned as OS threads via `spawn()` +/// - The main thread acts as coordinator: maintains the ready queue, +/// dispatches BuildJob messages to workers, and receives JobDone results +/// - No shared mutable state — all communication is via message passing +/// +/// Features: +/// - Ready queue with critical-path priority sorting (load-aware scheduling) +/// - Fine-grained dispatch: targets become ready immediately when their +/// last prerequisite completes (not batched) +/// - Worker pool reuse: N threads spawned once, reused for all targets +/// - Configurable job limits (-jN) +/// - Correct failure propagation: failed targets mark dependents as skipped +/// - .NOTPARALLEL support: targets run exclusively when all workers idle +/// - Output buffering via OutputManager for atomic per-target printing +module antelope.build.pool; + +import core.thread : Thread; +import std.concurrency : spawn, send, receive, receiveOnly, receiveTimeout, + Tid, thisTid, ownerTid; +import std.algorithm : sort; +import std.conv : to; + +import antelope.build.target; +import antelope.build.graph; +import antelope.build.dependency; +import antelope.build.output; +import antelope.build.executor; +import antelope.shell.process; +import antelope.shell.environment; +import antelope.filesystem.timestamps; +import antelope.compatibility.parallel; +import antelope.compatibility.vpath; +import antelope.diagnostics.output; + +// ── Messages ──────────────────────────────────────────────────────────── + +/// Sent from coordinator to worker: "build this target." +struct BuildJob +{ + string targetName; /// Target to build + immutable(string)[] expandedRecipe; /// Pre-expanded recipe lines + immutable(bool)[] ignoreErrors; /// Per-line: - prefix + immutable(bool)[] silent; /// Per-line: @ prefix + immutable(string)[] execEnv; /// KEY=VALUE environment + bool dryRun; /// If true, echo but don't execute +} + +/// Sent from worker to coordinator: "target built (or failed)." +struct JobDone +{ + Tid workerTid; /// Which worker completed + string targetName; /// Which target was built + bool success; /// True if all recipe lines succeeded + int exitCode; /// Last non-zero exit code (0 on success) + bool hadEcho; /// True if any recipe line was echoed (non-@) + immutable(string)[] stdoutLines; /// Captured stdout lines + immutable(string)[] stderrLines; /// Captured stderr lines +} + +/// Sent from coordinator to worker: "exit your loop." +struct Shutdown {} + +// ── WorkerPool ─────────────────────────────────────────────────────────── + +/// Manages parallel build execution. +/// +/// Usage: +/// ```d +/// auto pool = WorkerPool(config.jobs); +/// int exitCode = pool.build(graph, rootTargets, config, env, expander, output, vpath); +/// ``` +struct WorkerPool +{ +private: + uint numWorkers; /// Number of worker threads (= -j value) + Tid[] workerTids; /// Tids of spawned workers + bool started; /// Whether workers have been spawned + + // Build state (populated during build()) + DependencyGraph* graph; + ParallelConfig* parallelConfig; + OutputManager* outputMgr; + VPathConfig* vpathConfig; /// VPATH for needsRebuild in dequeueDependents + bool[string] notParallelSet; /// Targets marked .NOTPARALLEL + bool[string] failedSet; /// Targets that failed (for propagation) + bool[string] skippedSet; /// Targets blocked by failed prereqs + + // Ready queue (sorted by descending criticalWeight) + // Stored as indices into graph.targets + size_t[] readyQueue; + + // Dependency tracking + size_t[string] remainingDeps; + + // .JOBS throttle: tracks active workers per job-limit value. + size_t[size_t] activeByLimit; + + // Completed target counter (shared between build() and dequeueDependents). + size_t completedCount; + +public: + /// Create a worker pool with the given number of workers. + /// + /// If `numWorkers` is 0, it defaults to the number of CPU cores. + /// If `numWorkers` is 1, the build runs serially without spawning threads. + static WorkerPool create(uint numWorkers = 0) + { + import std.parallelism : totalCPUs; + WorkerPool pool; + if (numWorkers == 0) + pool.numWorkers = totalCPUs; + else + pool.numWorkers = numWorkers; + return pool; +} + +/// Jobserver file descriptors for cross-process token coordination. +/// readFd is passed to child processes via MAKEFLAGS; writeFd is +/// held by the parent to return tokens after job completion. +struct JobserverPipe +{ + int readFd = -1; /// Read end — child processes consume tokens here + int writeFd = -1; /// Write end — parent writes tokens back on completion + bool active; /// Whether the jobserver is operational +} + +/// Create a jobserver pipe with `nTokens` initial tokens. +/// +/// Writes N bytes to the pipe so that up to N jobs can run +/// concurrently across recursive $(MAKE) invocations. Each job +/// reads one byte before starting; the byte is written back on +/// completion. +static JobserverPipe createJobserverPipe(uint nTokens) +{ + version (Posix) + { + import core.sys.posix.unistd : pipe, read, write, close; + import core.sys.posix.fcntl : fcntl, F_SETFD, FD_CLOEXEC; + + JobserverPipe js; + int[2] fds; + + if (pipe(fds) != 0) + return js; + + // Write end: mark close-on-exec so child processes only + // inherit the read end. + fcntl(fds[1], F_SETFD, FD_CLOEXEC); + + js.readFd = fds[0]; + js.writeFd = fds[1]; + js.active = true; + + // Seed the pipe with N tokens (one byte each). + ubyte token = 0; + for (uint i = 0; i < nTokens; i++) + write(js.writeFd, &token, 1); + + return js; + } + else + { + // Non-POSIX: jobserver not supported. + return JobserverPipe(); + } +} + +/// Consume one token from the jobserver pipe (blocking). +/// Returns true if a token was acquired, false on error. +private static bool acquireJobserverToken(int readFd) +{ + version (Posix) + { + import core.sys.posix.unistd : read; + ubyte token; + return read(readFd, &token, 1) == 1; + } + else + return false; +} + +/// Return one token to the jobserver pipe. +private static bool releaseJobserverToken(int writeFd) +{ + version (Posix) + { + import core.sys.posix.unistd : write; + ubyte token = 0; + return write(writeFd, &token, 1) == 1; + } + else + return false; +} + + /// Run the build for the given root targets. + /// + /// Params: + /// graph = Dependency graph with all targets (mutated: runtime state fields) + /// roots = Root target names to build (e.g., ["all"]) + /// config = Parallel config (jobs, notParallelTargets, output sync mode) + /// env = Build environment (passed to recipe subprocesses) + /// expander = Variable expansion delegate + /// output = Output buffer manager + /// vpath = VPATH config (for needsRebuild checks) + /// baseExecEnv = KEY=VALUE pairs added to every job's env (e.g., SHELL, MAKEFLAGS) + /// dryRun = Print commands without executing + /// silent = Suppress command echoing + /// + /// Returns: exit code (0 = success, non-zero = failure). + int build( + ref DependencyGraph graph, + string[] roots, + ref ParallelConfig config, + Environment* env, + string delegate(string, string, string[], string) expander, + OutputManager* output, + VPathConfig* vpath, + string[] baseExecEnv = [], + bool dryRun = false, + bool silent = false) + { + import antelope.filesystem.timestamps; + + this.graph = &graph; + this.parallelConfig = &config; + this.outputMgr = output; + this.vpathConfig = vpath; + + // Determine actual worker count. + uint nWorkers = config.jobs; + if (nWorkers == 0) + { + import std.parallelism : totalCPUs; + nWorkers = totalCPUs; + } + + // Reset per-build state. + readyQueue = []; + failedSet = null; + skippedSet = null; + remainingDeps = null; + + // Serial mode shortcut. + if (nWorkers <= 1) + return buildSerial(graph, roots, config, env, expander, output, + vpath, baseExecEnv, dryRun, silent); + + // Build the combined transitive closure for all root targets. + bool[string] inClosure; + foreach (root; roots) + { + auto closure = graph.transitiveClosure(root); + foreach (name; closure) + inClosure[name] = true; + } + + if (inClosure.length == 0) + return 0; + + // Set up scheduling state. + graph.resetSchedulingState(); + graph.buildReverseEdges(); + + import antelope.build.dependency; + foreach (root; roots) + computeCriticalWeights(graph, root); + + graph.computeRemainingDeps(); + + // Copy remainingDeps for fast lookup BEFORE the init loop so + // that dequeueDependents (called for up-to-date targets during + // initial ready-queue construction) can read from it. + remainingDeps = null; + foreach (ref t; graph.targets) + if (t.name in inClosure) + remainingDeps[t.name] = t.remainingDeps; + + // Populate initial ready queue: targets with remainingDeps == 0 + // that actually need building and are in the closure. + size_t[] initialReady; + foreach (i, ref t; graph.targets) + { + if (t.name !in inClosure) + continue; + if (t.remainingDeps != 0) + continue; + if (!needsRebuild(t.name, t.prerequisites, + &graph.phonyTargets, vpath, &t.orderOnlyPrereqs)) + { + t.state = BuildState.completed; + // Notify dependents so they can become ready. + dequeueDependents(t.name); + continue; + } + initialReady ~= i; + } + + // Sort initial-ready targets by descending critical weight. + initialReady.sort!((a, b) => + graph.targets[a].criticalWeight > graph.targets[b].criticalWeight); + + // Merge initial-ready targets with any targets that became ready + // during the init loop (via dequeueDependents for up-to-date + // prerequisites). Initial-ready goes first (already filtered + // by needsRebuild), then dequeueDependents-added targets. + readyQueue = initialReady ~ readyQueue; + + size_t totalTargets = 0; + foreach (i, ref t; graph.targets) + if (t.name in inClosure && t.state != BuildState.completed) + totalTargets++; + + // Spawn worker threads. + workerTids.length = 0; + for (uint i = 0; i < nWorkers; i++) + { + auto tid = spawn(&workerFunc); + workerTids ~= tid; + } + this.numWorkers = nWorkers; + this.started = true; + + // Seed idle worker queue: all workers start idle. + Tid[] idleWorkers = workerTids.dup; + + // Send initial batch of jobs (respect .JOBS and .NOTPARALLEL limits). + completedCount = 0; + while (readyQueue.length > 0 && idleWorkers.length > 0) + { + size_t idx = readyQueue[0]; + + // NOTPARALLEL: only dispatch when all other workers are idle. + if (idx < graph.targets.length && + graph.targets[idx].name in notParallelSet && + idleWorkers.length != workerTids.length) + break; + + // .JOBS limit: throttle if this target has a job limit. + if (idx < graph.targets.length && + graph.targets[idx].jobLimit > 0) + { + size_t busy = workerTids.length - idleWorkers.length; + if (busy >= graph.targets[idx].jobLimit) + break; + } + + readyQueue = readyQueue[1 .. $]; + auto job = makeBuildJob(idx, env, expander, baseExecEnv, dryRun); + if (job.expandedRecipe.length > 0) + { + graph.targets[idx].state = BuildState.running; + send(idleWorkers[$ - 1], job); + idleWorkers = idleWorkers[0 .. $ - 1]; + if (graph.targets[idx].jobLimit > 0) + activeByLimit[graph.targets[idx].jobLimit]++; + } + else + { + // Target with no recipe: mark complete immediately. + graph.targets[idx].state = BuildState.completed; + dequeueDependents(graph.targets[idx].name); + completedCount++; + } + } + + // Coordinator loop: dispatch → receive → process → repeat. + bool hasFailure = false; + + while (completedCount < totalTargets) + { + // Phase 1: Dispatch as many ready targets as possible + // to idle workers. + while (readyQueue.length > 0 && idleWorkers.length > 0) + { + size_t idx = readyQueue[0]; + + // NOTPARALLEL: only dispatch when all other workers are idle. + if (idx < graph.targets.length && + graph.targets[idx].name in notParallelSet && + idleWorkers.length != workerTids.length) + break; + + // .JOBS limit: throttle if this target has a job limit. + // Tracks only workers running .JOBS-limited targets of + // the same limit value, not total busy workers. + if (idx < graph.targets.length && + graph.targets[idx].jobLimit > 0) + { + size_t limit = graph.targets[idx].jobLimit; + auto countPtr = limit in activeByLimit; + size_t active = countPtr ? *countPtr : 0; + if (active >= limit) + break; + } + + readyQueue = readyQueue[1 .. $]; + auto job = makeBuildJob(idx, env, expander, baseExecEnv, dryRun); + if (job.expandedRecipe.length > 0) + { + graph.targets[idx].state = BuildState.running; + send(idleWorkers[$ - 1], job); + idleWorkers = idleWorkers[0 .. $ - 1]; + if (graph.targets[idx].jobLimit > 0) + activeByLimit[graph.targets[idx].jobLimit]++; + } + else + { + // Target with no recipe: complete immediately. + graph.targets[idx].state = BuildState.completed; + dequeueDependents(graph.targets[idx].name); + completedCount++; + } + } + + // Phase 2: Check termination. + if (idleWorkers.length == workerTids.length) + { + // All workers idle. If work remains, targets are + // blocked (waiting for failed/skipped prereqs). + if (completedCount < totalTargets) + { + foreach (ref t; graph.targets) + { + if (t.name !in inClosure) + continue; + if (t.state == BuildState.pending) + { + t.state = BuildState.skipped; + skippedSet[t.name] = true; + completedCount++; + if (output && output.buffered) + output.flush(t.name); + } + } + } + break; + } + + // Phase 3: Wait for a worker result. + auto done = receiveOnly!JobDone(); + + // Worker is now idle. + idleWorkers ~= done.workerTid; + + // Buffer output into OutputManager. + if (output) + { + foreach (line; done.stdoutLines) + output.bufferStdout(done.targetName, cast(string) line); + foreach (line; done.stderrLines) + output.bufferStderr(done.targetName, cast(string) line); + if (done.hadEcho) + output.markEchoed(done.targetName); + } + + // Phase 4: Process result. + auto tp = graph.findTarget(done.targetName); + if (tp !is null) + { + // Decrement .JOBS throttle counter if this target had a limit. + if (tp.jobLimit > 0) + { + auto countPtr = tp.jobLimit in activeByLimit; + if (countPtr && *countPtr > 0) + (*countPtr)--; + } + + if (done.success) + { + tp.state = BuildState.completed; + log(LogLevel.dbg, "[" ~ done.targetName ~ "] completed"); + dequeueDependents(done.targetName); + } + else + { + tp.state = BuildState.failed; + failedSet[done.targetName] = true; + hasFailure = true; + log(LogLevel.dbg, "[" ~ done.targetName ~ + "] FAILED (exit " ~ done.exitCode.to!string ~ ")"); + propagateFailure(done.targetName, inClosure); + } + } + + // Flush buffered output. + if (output && output.buffered) + output.flush(done.targetName); + + completedCount++; + // Loop back to Phase 1 (dispatch newly-ready targets). + } + + // Shutdown all workers. + foreach (tid; workerTids) + { + try { send(tid, Shutdown()); } catch (Exception) {} + } + + // Collect any late messages (workers may have sent results + // that haven't been received yet due to timing). + // Use a short timeout to drain the queue. + import core.time : dur; + while (true) + { + auto msg = receiveTimeout(dur!"msecs"(100), (JobDone d) => true); + if (!msg) + break; + } + + workerTids = []; + started = false; + + return hasFailure ? 1 : 0; + } + +private: + /// Serial fallback for -j1 or single-worker builds. + int buildSerial( + ref DependencyGraph graph, + string[] roots, + ref ParallelConfig config, + Environment* env, + string delegate(string, string, string[], string) expander, + OutputManager* output, + VPathConfig* vpath, + string[] baseExecEnv, + bool dryRun, + bool silent) + { + import antelope.filesystem.timestamps : needsRebuild; + + bool hasFailure; + + foreach (root; roots) + { + auto batches = resolveDependencies(graph, root); + foreach (batch; batches) + { + foreach (ref t; batch) + { + if (!needsRebuild(t.name, t.prerequisites, + &graph.phonyTargets, vpath, &t.orderOnlyPrereqs)) + continue; + + string[] execEnv = baseExecEnv.dup; + + auto result = executeTarget(t, execEnv, expander, + output, dryRun, silent); + if (output && output.buffered) + output.flush(t.name); + + if (!result.success) + { + hasFailure = true; + goto done; + } + } + } + } + + done: + return hasFailure ? 1 : 0; + } + + /// Worker thread function. + static void workerFunc() + { + import std.string : stripLeft; + bool running = true; + + while (running) + { + receive( + (BuildJob job) { + JobDone done; + done.workerTid = thisTid; + done.targetName = job.targetName; + done.success = true; + done.exitCode = 0; + done.hadEcho = false; + + // Build mutable output buffers, freeze before sending. + string[] outLines; + string[] errLines; + + if (job.expandedRecipe.length == 0) + { + done.stdoutLines = outLines.idup; + done.stderrLines = errLines.idup; + send(ownerTid, done); + return; + } + + foreach (i, line; job.expandedRecipe) + { + // line is immutable(string); cast to string for stdlib. + string sline = cast(string) line; + if (sline.stripLeft.length == 0) + continue; + + bool ignoreErrors = i < job.ignoreErrors.length + ? cast(bool) job.ignoreErrors[i] : false; + bool silent = i < job.silent.length + ? cast(bool) job.silent[i] : false; + + // Echo + if (!silent) + { + done.hadEcho = true; + outLines ~= sline; + } + + // Dry run: skip actual execution. + if (job.dryRun) + continue; + + // Execute. Cast execEnv back to mutable for runProcessPiped. + auto ph = runProcessPiped(sline, cast(string[]) job.execEnv); + + // Read pipes. + try + { + import std.string : chomp; + + foreach (pl; ph.stdoutPipe.byLine) + { + string s = pl.chomp().idup; + outLines ~= s; + } + + foreach (pl; ph.stderrPipe.byLine) + { + string s = pl.chomp().idup; + errLines ~= s; + } + } + catch (Exception e) + { + // Log pipe read errors but don't abort the build. + log(LogLevel.dbg, "[" ~ job.targetName ~ + "] pipe read error: " ~ e.msg); + } + + int code = ph.waitFor(); + + if (code != 0 && !ignoreErrors) + { + ph.closePipes(); + done.success = false; + done.exitCode = code; + done.stdoutLines = outLines.idup; + done.stderrLines = errLines.idup; + send(ownerTid, done); + return; + } + + ph.closePipes(); + } + + done.stdoutLines = outLines.idup; + done.stderrLines = errLines.idup; + send(ownerTid, done); + }, + (Shutdown _) { + running = false; + } + ); + } + } + + /// Create a BuildJob for the target at graph index `idx`. + BuildJob makeBuildJob( + size_t idx, + Environment* env, + string delegate(string, string, string[], string) expander, + string[] baseExecEnv, + bool dryRun) + { + auto t = &graph.targets[idx]; + + // Build mutable arrays, then freeze to immutable for sending. + string[] recipeLines; + bool[] ignoreErrs; + bool[] silents; + + foreach (line; t.recipe) + { + // Expand variables in the recipe. + string expanded = expander(line, t.name, + t.prerequisites, t.stem); + + // Strip and classify prefix characters. + import std.string : stripLeft; + string trimmed = expanded.stripLeft(); + bool ignoreErrors; + bool silent; + + if (trimmed.length > 0) + { + bool stripping = true; + while (stripping && trimmed.length > 0) + { + stripping = false; + switch (trimmed[0]) + { + case '@': + silent = true; + trimmed = trimmed[1 .. $]; + stripping = true; + break; + case '-': + ignoreErrors = true; + trimmed = trimmed[1 .. $]; + stripping = true; + break; + case '+': + trimmed = trimmed[1 .. $]; + stripping = true; + break; + default: + break; + } + } + } + + if (trimmed.length == 0) + continue; + + recipeLines ~= trimmed; + ignoreErrs ~= ignoreErrors; + silents ~= silent; + } + + // Build execEnv. + string[] execEnvArr = baseExecEnv.dup; + + // Freeze arrays to immutable for std.concurrency message passing. + BuildJob job; + job.targetName = t.name; + job.expandedRecipe = recipeLines.idup; + job.ignoreErrors = ignoreErrs.idup; + job.silent = silents.idup; + job.execEnv = execEnvArr.idup; + job.dryRun = dryRun; + + return job; + } + + /// Decrement remainingDeps for all dependents of `completedTarget`. + /// Any dependent that reaches 0 remaining deps is added to the + /// ready queue (sorted by descending critical weight). + void dequeueDependents(string completedTarget) + { + auto tp = graph.findTarget(completedTarget); + if (tp is null) + return; + + foreach (depName; tp.dependents) + { + // Skip if not in our tracking (could be outside closure). + auto depPtr = depName in remainingDeps; + if (depPtr is null) + continue; + + if (*depPtr == 0) + continue; // Already ready or completed + + (*depPtr)--; + + if (*depPtr == 0 && !(depName in skippedSet)) + { + // Target is now ready — check if it needs building. + auto dep = graph.findTarget(depName); + if (dep is null || dep.state != BuildState.pending) + continue; + + // Check up-to-date: targets that become ready via + // dequeueDependents were NOT filtered during the init + // loop (which only checks initially-zero-dep targets). + import antelope.filesystem.timestamps : needsRebuild; + if (!needsRebuild(dep.name, dep.prerequisites, + &graph.phonyTargets, vpathConfig, &dep.orderOnlyPrereqs)) + { + dep.state = BuildState.completed; + completedCount++; + dequeueDependents(dep.name); + continue; + } + + // Insert sorted by descending critical weight. + bool inserted; + foreach (i, qi; readyQueue) + { + if (dep.criticalWeight > graph.targets[qi].criticalWeight) + { + readyQueue = readyQueue[0 .. i] ~ + [cast(size_t)(dep - graph.targets.ptr)] ~ + readyQueue[i .. $]; + inserted = true; + break; + } + } + if (!inserted) + readyQueue ~= cast(size_t)(dep - graph.targets.ptr); + } + } + } + + /// Mark all dependents of a failed target as skipped. + /// Recursively propagates: if A depends on B and B fails, + /// A is skipped; if C depends on A, C is also skipped. + void propagateFailure(string failedTarget, ref bool[string] inClosure) + { + import std.algorithm : canFind; + + string[] stack = [failedTarget]; + + while (stack.length > 0) + { + string current = stack[$ - 1]; + stack = stack[0 .. $ - 1]; + + auto tp = graph.findTarget(current); + if (tp is null) + continue; + + foreach (depName; tp.dependents) + { + if (depName in skippedSet || depName in failedSet) + continue; + if (depName !in inClosure) + continue; + + auto dep = graph.findTarget(depName); + if (dep is null) + continue; + + dep.state = BuildState.skipped; + skippedSet[depName] = true; + stack ~= depName; + } + } + } +} + +// ── Unittests ──────────────────────────────────────────────────────────── + +/// +unittest +{ + // Build a trivial graph with one target (no recipe). + DependencyGraph g; + g.addTarget(Target("leaf", TargetKind.file, [], [])); + + g.buildReverseEdges(); + g.computeRemainingDeps(); + computeCriticalWeights(g, "leaf"); + + ParallelConfig pc; + pc.jobs = 2; + + auto om = new OutputManager(); + + string expand(string ln, string tn, string[] pr, string st) { return ln; } + + auto pool = WorkerPool.create(2); + int code = pool.build(g, ["leaf"], pc, null, &expand, &om, null); + assert(code == 0); + assert(g.findTarget("leaf").state == BuildState.completed); +} + +/// +unittest +{ + // Chain: a → b → c (all no recipe, always succeed). + DependencyGraph g; + g.addTarget(Target("c", TargetKind.file, [], [])); + g.addTarget(Target("b", TargetKind.file, ["c"], [])); + g.addTarget(Target("a", TargetKind.file, ["b"], [])); + + // Ensure all are in the closure. + g.buildReverseEdges(); + g.computeRemainingDeps(); + computeCriticalWeights(g, "a"); + + ParallelConfig pc; + pc.jobs = 2; + + auto om = new OutputManager(); + string expand(string ln, string tn, string[] pr, string st) { return ln; } + + auto pool = WorkerPool.create(2); + int code = pool.build(g, ["a"], pc, null, &expand, &om, null); + assert(code == 0); + assert(g.findTarget("a").state == BuildState.completed); + assert(g.findTarget("b").state == BuildState.completed); + assert(g.findTarget("c").state == BuildState.completed); +} + +// Regression: serial mode (jobs=1) should work. +unittest +{ + DependencyGraph g; + g.addTarget(Target("x", TargetKind.file, [], [])); + + g.buildReverseEdges(); + g.computeRemainingDeps(); + computeCriticalWeights(g, "x"); + + ParallelConfig pc; + pc.jobs = 1; + + auto om = new OutputManager(); + string expand(string ln, string tn, string[] pr, string st) { return ln; } + + auto pool = WorkerPool.create(1); + int code = pool.build(g, ["x"], pc, null, &expand, &om, null); + assert(code == 0); +} diff --git a/source/antelope/build/target.d b/source/antelope/build/target.d index 669bd54..758fac6 100644 --- a/source/antelope/build/target.d +++ b/source/antelope/build/target.d @@ -9,6 +9,20 @@ enum TargetKind intermediate, } +/// Runtime scheduling state for parallel builds. +/// +/// Tracks where a target is in the build lifecycle so the worker pool +/// can make dispatch decisions. +enum BuildState +{ + pending, /// Not yet ready — outstanding in-graph prerequisites + ready, /// All prereqs satisfied, can be dispatched to a worker + running, /// Currently being built by a worker thread + completed, /// Built successfully + failed, /// Build failed — dependents will be skipped + skipped, /// Blocked by a failed prerequisite +} + /// A single build target. struct Target { @@ -18,4 +32,33 @@ struct Target string[] recipe; /// Shell commands to build this target string[] orderOnlyPrereqs; /// Order-only prerequisites (| — must exist, no rebuild trigger) string stem; /// Pattern/suffix rule stem for $* expansion + + // ── Runtime scheduling fields ─────────────────────────────────────── + + /// Current build state (mutated by the worker pool coordinator). + BuildState state = BuildState.pending; + + /// How many in-graph prerequisites still need to complete before this + /// target becomes `ready`. Set by `DependencyGraph.computeRemainingDeps()` + /// before the build begins. + size_t remainingDeps; + + /// Critical-path weight for load-aware scheduling. + /// + /// Computed as: `recipe.length + max(successor.criticalWeight)`. + /// Leaf nodes (no in-graph successors) have weight = `recipe.length`. + /// Higher weight → on the critical path → should be scheduled first + /// when multiple targets are in the ready queue. + size_t criticalWeight; + + /// Reverse edges: names of targets that list this target as a prerequisite. + /// Populated by `DependencyGraph.buildReverseEdges()` before scheduling. + /// When this target completes, every name in this list will have its + /// `remainingDeps` decremented. + string[] dependents; + + /// Per-target job limit for .JOBS special target (native mode only). + /// 0 = use the global pool limit. Non-zero = maximum concurrent + /// jobs allowed for this specific target's recipe group. + size_t jobLimit; } diff --git a/source/antelope/cli/subcommands.d b/source/antelope/cli/subcommands.d index cdd0d2f..815c207 100644 --- a/source/antelope/cli/subcommands.d +++ b/source/antelope/cli/subcommands.d @@ -29,6 +29,8 @@ int dispatchSubcommand(CliConfig config) /// Execute the build (default subcommand). /// /// Full pipeline: find build file → parse → evaluate → schedule → execute. +/// The schedule/execute phases now use the parallel WorkerPool for +/// dependency-aware concurrent builds when `-j > 1`. int runBuild(CliConfig config) { import antelope.parser.parser; @@ -40,8 +42,12 @@ int runBuild(CliConfig config) import antelope.build.dependency; import antelope.build.scheduler; import antelope.build.executor; + import antelope.build.pool; + import antelope.build.output; import antelope.shell.environment; import antelope.filesystem.timestamps; + import antelope.compatibility.parallel; + import antelope.compatibility.submake; // Set log level if (config.debugMode) @@ -104,10 +110,12 @@ int runBuild(CliConfig config) // Set MAKE to the antelope binary path for $(MAKE) in recipes. // Include -gnu so recursive sub-makes inherit GNU compat mode. + // config.file is shell-quoted to prevent injection when $(MAKE) + // is used in recipes. import std.file : thisExePath; string makeCmd = thisExePath(); if (config.gnuMode) makeCmd ~= " -gnu"; - if (config.file.length > 0) makeCmd ~= " -f " ~ config.file; + if (config.file.length > 0) makeCmd ~= " -f '" ~ escapeShell(config.file) ~ "'"; env.set("MAKE", makeCmd); // Set MAKECMDGOALS from command-line targets (autotools compat) @@ -171,7 +179,13 @@ int runBuild(CliConfig config) return 1; } + // Process special targets after evaluation populates the graph. + (*graph).handlePhony(); + (*graph).handleWait(); + (*graph).handleJobs(); + // Check for cycle errors + (*graph).detectCycles(); if (graph.cycleErrors.length > 0) { foreach (err; graph.cycleErrors) @@ -217,7 +231,7 @@ int runBuild(CliConfig config) else buildTargets = [graph.targets[0].name]; - // .PHONY targets are tracked in the graph automatically via handlePhony() + // .PHONY + .WAIT + .JOBS targets are processed above via handlePhony/handleWait/handleJobs. // --- VPATH configuration (GNU Make compat) --- import antelope.compatibility.vpath; @@ -252,16 +266,14 @@ int runBuild(CliConfig config) } } - // Build each requested target - int exitCode = 0; + // --- Resolve implicit targets (GNU Make compat) --- + // Targets requested on the command line might not exist in the graph + // yet — they may be defined only via pattern/suffix rules. + // We create stubs and run implicit rule resolution so they can be built. foreach (targetName; buildTargets) { if (!graph.hasTarget(targetName)) { - // Target not explicitly defined — try to create it from - // implicit rules (suffix rules, pattern rules). Autotools - // Makefiles invoke $(MAKE) with targets like "be.gmo" that - // are only defined via suffix rules (e.g., .po.gmo:). import antelope.build.target; Target stub; stub.name = targetName; @@ -270,93 +282,107 @@ int runBuild(CliConfig config) import antelope.evaluator.evaluator : resolveImplicitRules; resolveImplicitRules(*graph, env); - - if (!graph.hasTarget(targetName) || - graph.findTarget(targetName).recipe.length == 0) - { - log(LogLevel.normal, "antelope: *** No rule to make target '" ~ - targetName ~ "'. Stop."); - return 1; - } } + } - // Resolve dependencies and check what needs building - auto batches = resolveDependencies(*graph, targetName); + // --- Set up parallel execution config --- + ParallelConfig parallelCfg; + parallelCfg.jobs = config.jobs; + parallelCfg.heuristic = SchedulingHeuristic.criticalPath; - import std.stdio; - if (targetName == "libgnu.a" || targetName == "all") { - stderr.writefln(" batches=%d", batches.length); - foreach (i, batch; batches) { - size_t w; - foreach (ref t; batch) if (t.recipe.length > 0) w++; - stderr.writefln(" batch[%d]: %d targets, %d with recipe", i, batch.length, w); - } - } - - bool builtSomething = false; - foreach (batch; batches) + // Check for .NOTPARALLEL targets in the graph. + if (graph.hasTarget(".NOTPARALLEL")) + { + auto np = graph.findTarget(".NOTPARALLEL"); + if (np !is null) { - foreach (ref t; batch) + foreach (name; np.prerequisites) + parallelCfg.notParallelTargets ~= name; + } + } + + // Output mode: buffer when parallel, live when serial. + auto outputMgr = new OutputManager(); + if (config.jobs > 1 || config.jobs == 0) + { + outputMgr.buffered = true; + parallelCfg.outputSync = OutputSyncMode.target; + } + + // --- Build base execution environment --- + string[] baseExecEnv; + if (env.hasKey("SHELL")) + baseExecEnv ~= "SHELL=" ~ env.get("SHELL"); + + // Create jobserver pipe for cross-process token coordination. + // Only created when parallel build is active (-j > 1). + // Create jobserver pipe for cross-process token coordination. + // Only created when parallel build is active (-j > 1). + import antelope.build.pool : WorkerPool; + WorkerPool.JobserverPipe jsPipe; + if (config.jobs > 1) + jsPipe = WorkerPool.createJobserverPipe(config.jobs); + + // Serialize MAKEFLAGS for recursive $(MAKE) calls. + string makeFlags = serializeMakeFlags(config, jsPipe.readFd, jsPipe.writeFd); + if (makeFlags.length > 0) + baseExecEnv ~= "MAKEFLAGS=" ~ makeFlags; + + // --- Variable expansion delegate --- + // Captured by the pool and called per-target during job construction. + // This delegates to the existing expand() function from the evaluator, + // threading through the global Environment and target context. + string expander(string line, string targetName, + string[] prerequisites, string stem) + { + return expand(line, env, targetName, prerequisites, stem); + } + + // --- Dispatch build --- + auto pool = WorkerPool.create(config.jobs); + int exitCode = pool.build( + *graph, buildTargets, parallelCfg, env, + &expander, &outputMgr, &vpath, + baseExecEnv, config.dryRun, false); + + // Report up-to-date targets (tracks targets that had no work). + if (exitCode == 0) + { + bool anyBuilt; + foreach (targetName; buildTargets) + { + auto tp = graph.findTarget(targetName); + if (tp !is null && tp.state == BuildState.completed) { - if (!needsRebuild(t.name, t.prerequisites, - &graph.phonyTargets, &vpath, &t.orderOnlyPrereqs)) - continue; - - builtSomething = true; - - // Execute recipe lines - foreach (recipeLine; t.recipe) + if (tp.recipe.length == 0 && !outputMgr.hasEchoed(targetName)) { - // Expand variables in the recipe - string expanded = expand(recipeLine, env, t.name, - t.prerequisites, t.stem); - - // Print the command unless silent (@ prefix) - import std.string : stripLeft; - string trimmed = recipeLine.stripLeft(); - if (config.dryRun || config.debugMode || recipeLine.length == 0 || - (trimmed.length > 0 && trimmed[0] != '@')) - { - log(LogLevel.normal, expanded); - } - - // Build environment for recipe execution - // (propagate SHELL from Makefile if set) - string[] execEnv; - if (env.hasKey("SHELL")) - execEnv ~= "SHELL=" ~ env.get("SHELL"); - - // Serialize MAKEFLAGS for recursive $(MAKE) calls - import antelope.compatibility.submake; - string makeFlags = serializeMakeFlags(config); - if (makeFlags.length > 0) - execEnv ~= "MAKEFLAGS=" ~ makeFlags; - - // Execute unless dry run - if (!config.dryRun) - { - auto result = execute(expanded, execEnv); - if (!result.success) - { - log(LogLevel.normal, "antelope: *** [" ~ t.name ~ - "] Error " ~ result.exitCode.to!string); - return result.exitCode; - } - } + // Target was up-to-date or had no recipe. } + else if (!outputMgr.hasEchoed(targetName)) + { + log(LogLevel.normal, "antelope: '" ~ targetName ~ + "' is up to date."); + } + anyBuilt = true; } } - - if (!builtSomething) + if (!anyBuilt) { - log(LogLevel.normal, "antelope: '" ~ targetName ~ - "' is up to date."); + // Check if nothing needed building (all targets already up to date). } } return exitCode; } +/// Escape a string for single-quoted shell usage. +/// Replaces each `'` with `'\''` so the value can be wrapped in single quotes. +private string escapeShell(string s) +{ + import std.array : replace; + return s.replace("'", "'\\''"); +} + /// Find which build file to use based on mode and config. private string findBuildFile(CliConfig config) { diff --git a/source/antelope/compatibility/parallel.d b/source/antelope/compatibility/parallel.d index 298df9b..1f230bd 100644 --- a/source/antelope/compatibility/parallel.d +++ b/source/antelope/compatibility/parallel.d @@ -18,13 +18,45 @@ enum ParallelSpecialTarget jobs, /// .JOBS } +/// Output synchronisation mode for parallel builds. +/// +/// GNU Make 4.0+ supports `--output-sync` with four modes. +enum OutputSyncMode +{ + none, /// No synchronisation — output may be interleaved (default) + target, /// Buffer output per target, print atomically on completion + line, /// Buffer output per line, print atomically per line + recurse, /// Buffer output per recursive make invocation +} + +/// Which scheduling heuristic to use when multiple targets are ready. +enum SchedulingHeuristic +{ + fifo, /// First-in-first-out — build order is topological order + criticalPath, /// Prioritise targets on the critical path (minimise total build time) +} + /// Parallel execution configuration. struct ParallelConfig { /// Maximum parallel jobs (0 = unlimited, 1 = serial). uint jobs = 1; - /// Targets excluded from parallel builds. + + /// Targets excluded from parallel builds (.NOTPARALLEL). + /// These always run serially, even when -j > 1. string[] notParallelTargets; + /// Whether to use jobserver protocol for sub-makes. bool useJobserver = true; + + /// How to synchronise output from parallel jobs. + OutputSyncMode outputSync = OutputSyncMode.none; + + /// Which heuristic to use for ordering the ready queue. + SchedulingHeuristic heuristic = SchedulingHeuristic.criticalPath; + + /// Timeout in seconds for individual recipe lines (0 = no timeout). + /// GNU Make does not natively support job timeouts; this is an + /// Antelope extension. + uint jobTimeoutSecs = 0; } diff --git a/source/antelope/compatibility/submake.d b/source/antelope/compatibility/submake.d index c92c887..ca8c1dd 100644 --- a/source/antelope/compatibility/submake.d +++ b/source/antelope/compatibility/submake.d @@ -14,7 +14,6 @@ module antelope.compatibility.submake; import antelope.cli.args; import std.conv : to; import std.string : strip; -import std.string : strip; /// Sub-make communication options. struct SubMakeConfig @@ -36,14 +35,30 @@ struct SubMakeConfig /// MAKEFLAGS environment variable for recursive $(MAKE) invocations. /// /// Serialized flags: -/// -j — parallel job count (only when jobs > 1) -/// -n — dry run -/// -P — POSIX conformance mode -/// -d — debug output -/// VAR=val — command-line variable overrides +/// -j — parallel job count (only when jobs > 1) +/// -n — dry run +/// -P — POSIX conformance mode +/// -d — debug output +/// --jobserver-auth=R,W — jobserver pipe file descriptors (when provided) +/// VAR=val — command-line variable overrides /// /// Returns: a space-delimited MAKEFLAGS string, or "" if no flags are active. string serializeMakeFlags(CliConfig config) +{ + return serializeMakeFlagsImpl(config, 0, 0); +} + +/// Serialize MAKEFLAGS including jobserver pipe descriptors. +/// +/// When `readFd` and `writeFd` are non-zero, appends +/// `--jobserver-auth=,` to the MAKEFLAGS string +/// so sub-make processes can participate in the shared job pool. +string serializeMakeFlags(CliConfig config, int readFd, int writeFd) +{ + return serializeMakeFlagsImpl(config, readFd, writeFd); +} + +private string serializeMakeFlagsImpl(CliConfig config, int readFd, int writeFd) { string flags; @@ -59,5 +74,14 @@ string serializeMakeFlags(CliConfig config) flags ~= " -P"; if (config.debugMode) flags ~= " -d"; + + // Jobserver pipe file descriptors for recursive make coordination. + // Only included when the pool has created a jobserver pipe. + if (readFd > 0 && writeFd > 0) + { + import std.conv : to; + flags ~= " --jobserver-auth=" ~ readFd.to!string ~ "," ~ writeFd.to!string; + } + return flags.strip; } diff --git a/source/antelope/shell/process.d b/source/antelope/shell/process.d index 1eca9a5..7077ada 100644 --- a/source/antelope/shell/process.d +++ b/source/antelope/shell/process.d @@ -4,20 +4,118 @@ /// Commands are run via `/bin/sh -c` on POSIX systems. module antelope.shell.process; -import std.process : spawnProcess, wait; +import std.process : spawnProcess, spawnShell, wait, Pid, Pipe, pipeShell, + Redirect, ProcessPipes, Config; import std.string : indexOf; +import std.stdio : File; -/// Run a command via shell and return its exit code. +/// Handle to a running piped subprocess. +/// +/// Created by `runProcessPiped()`, this lets the caller read +/// stdout/stderr asynchronously and wait for completion. +struct ProcessHandle +{ + Pid pid; /// Process ID + File stdoutPipe; /// File for reading process stdout + File stderrPipe; /// File for reading process stderr + string command; /// The command that was executed (for error messages) + + /// Wait for the process to finish and return its exit code. + /// Returns: the exit code, or -1 if waiting failed. + int waitFor() + { + try + { + return wait(pid); + } + catch (Exception) + { + return -1; + } + } + + /// Close the stdout and stderr pipes. + /// Must be called after reading all output to avoid fd leaks. + void closePipes() + { + try { stdoutPipe.close(); } catch (Exception) {} + try { stderrPipe.close(); } catch (Exception) {} + } +} + +/// Run a command via shell with piped stdout and stderr. +/// +/// Unlike `runProcess()` which blocks and inherits parent fds, +/// this variant captures stdout and stderr into pipes so output +/// can be buffered and printed atomically by the caller. +/// +/// Params: +/// command = The raw shell command to execute +/// environment = Optional KEY=VALUE pairs for the process environment +/// +/// Returns: A ProcessHandle that can be used to read output and wait. +/// +/// Throws: Exception if process spawning fails. +ProcessHandle runProcessPiped(string command, string[] environment = []) +{ + // Determine shell — respect SHELL variable, default to /bin/sh + string shell = "/bin/sh"; + foreach (env; environment) + { + if (env.length > 6 && env[0 .. 6] == "SHELL=") + { + shell = env[6 .. $]; + break; + } + } + + if (environment.length > 0) + { + string[string] envMap; + foreach (env; environment) + { + auto idx = env.indexOf('='); + if (idx != -1) + envMap[env[0 .. idx]] = env[idx + 1 .. $]; + } + + auto pipes = pipeShell(command, + Redirect.stdout | Redirect.stderr, + envMap, Config.none, null, shell); + + ProcessHandle h; + h.pid = pipes.pid; + h.stdoutPipe = pipes.stdout; + h.stderrPipe = pipes.stderr; + h.command = command; + return h; + } + else + { + auto pipes = pipeShell(command, + Redirect.stdout | Redirect.stderr, + null, Config.none, null, shell); + + ProcessHandle h; + h.pid = pipes.pid; + h.stdoutPipe = pipes.stdout; + h.stderrPipe = pipes.stderr; + h.command = command; + return h; + } +} + +/// Run a command via shell and return its exit code (blocking, no capture). +/// +/// This is the original synchronous variant — used for simple cases +/// and single-job builds where output capture is unnecessary. /// /// If `command` is empty, returns 0 immediately without spawning. /// The shell used is determined by the SHELL environment variable, /// defaulting to /bin/sh if not set. -/// If `environment` is non-empty, it is parsed as KEY=VALUE pairs and -/// passed as the process environment; otherwise the parent -/// process environment is inherited. -/// Stdout and stderr are inherited from the parent (not captured here). +/// /// Returns: the exit code of the command, or -1 if spawning failed. -int runProcess(string command, string[] environment) +int runProcess(string command, string[] environment = []) { if (command.length == 0) return 0; @@ -26,9 +124,9 @@ int runProcess(string command, string[] environment) string shell = "/bin/sh"; foreach (env; environment) { - if (env.length > 6 && env[0..6] == "SHELL=") + if (env.length > 6 && env[0 .. 6] == "SHELL=") { - shell = env[6..$]; + shell = env[6 .. $]; break; } } @@ -54,7 +152,7 @@ int runProcess(string command, string[] environment) return wait(pid); } } - catch (Exception e) + catch (Exception) { return -1; } @@ -65,3 +163,21 @@ unittest assert(runProcess("echo hello", []) == 0); assert(runProcess("exit 42", []) == 42); } + +/// Verify that `runProcessPiped` returns the same exit code as `runProcess`. +unittest +{ + auto h = runProcessPiped("echo hello", []); + int code = h.waitFor(); + assert(code == 0); + h.closePipes(); +} + +/// Test piped stderr capture via non-zero exit. +unittest +{ + auto h = runProcessPiped("exit 7", []); + int code = h.waitFor(); + assert(code == 7); + h.closePipes(); +} diff --git a/tests/build/graph_test.d b/tests/build/graph_test.d new file mode 100644 index 0000000..879dd76 --- /dev/null +++ b/tests/build/graph_test.d @@ -0,0 +1,35 @@ +/// Unit tests for the dependency graph construction. +/// +/// Tests DAG building from AST rules, cycle detection, and +/// topological ordering. +module antelope.tests.build.graph_test; + +import antelope.build.graph; +import antelope.build.target; + +/// Test constructing a simple linear dependency graph. +unittest +{ + Target[] targets = [ + Target("all", TargetKind.phony, ["hello"], []), + Target("hello", TargetKind.file, ["hello.o"], []), + Target("hello.o", TargetKind.file, ["hello.c"], []), + Target("hello.c", TargetKind.file, [], []), + ]; + + auto graph = DependencyGraph.fromTargets(targets); + assert(graph.nodes.length == 4); +} + +/// Test cycle detection. +unittest +{ + Target[] targets = [ + Target("a", TargetKind.file, ["b"], []), + Target("b", TargetKind.file, ["c"], []), + Target("c", TargetKind.file, ["a"], []), + ]; + + auto graph = DependencyGraph.fromTargets(targets); + assert(graph.hasCycle()); +} diff --git a/tests/compatibility/gnu_make_test.d b/tests/compatibility/gnu_make_test.d new file mode 100644 index 0000000..7a77d4d --- /dev/null +++ b/tests/compatibility/gnu_make_test.d @@ -0,0 +1,44 @@ +/// GNU Make compatibility conformance tests. +/// +/// These tests verify that Antelope in -gnu mode produces the same +/// output as GNU Make for a set of canonical Makefiles. Each test +/// runs both Antelope and GNU Make on the same input and compares +/// stdout, exit code, and files produced. +module antelope.tests.compatibility.gnu_make_test; + +import antelope.compatibility.gnu_make; +import antelope.compatibility.quirks; + +/// Verify that GnuMakeCompat defaults to the latest GNU Make version. +unittest +{ + auto compat = GnuMakeCompat.init; + assert(compat.targetVersion == GnuMakeVersion.v4_4); +} + +/// Verify key quirks are enabled by default. +unittest +{ + // Backslash-newline in comments is a known GNU Make quirk + // that many real-world Makefiles depend on. + assert(GnuQuirk.defaultQuirks.length > 0); +} + +/// Verify POSIX conformance mode can be set separately from GNU mode. +unittest +{ + auto compat = GnuMakeCompat.init; + auto posix = PosixCompat.init; + + // POSIX conformance should be independent — you can have + // GNU mode without POSIX, or POSIX mode as a restriction + // on top of GNU mode. + posix.mode = PosixConformance.gnu_mode; + assert(posix.mode == PosixConformance.gnu_mode); + + posix.mode = PosixConformance.posix_target; + assert(posix.mode == PosixConformance.posix_target); + + posix.mode = PosixConformance.strict; + assert(posix.mode == PosixConformance.strict); +} diff --git a/tests/evaluator/expansion_test.d b/tests/evaluator/expansion_test.d new file mode 100644 index 0000000..ae4e12d --- /dev/null +++ b/tests/evaluator/expansion_test.d @@ -0,0 +1,37 @@ +/// Unit tests for variable expansion in the evaluator. +/// +/// Tests recursive expansion, simple vs recursive assignment semantics, +/// and circular reference detection. +module antelope.tests.evaluator.expansion_test; + +import antelope.evaluator.expansion; +import antelope.shell.environment; + +/// Test simple variable reference expansion. +unittest +{ + auto env = Environment.init; + env.set("CC", "gcc"); + env.set("CFLAGS", "-Wall -O2"); + + auto result = expand(env, "$(CC) $(CFLAGS)"); + assert(result == "gcc -Wall -O2"); +} + +/// Test brace-delimited variable expansion. +unittest +{ + auto env = Environment.init; + env.set("VAR", "value"); + + auto result = expand(env, "${VAR}"); + assert(result == "value"); +} + +/// Test that undefined variables produce a warning but expand to empty. +unittest +{ + auto env = Environment.init; + auto result = expand(env, "$(UNDEFINED)"); + assert(result == ""); +} diff --git a/tests/parser/lexer_test.d b/tests/parser/lexer_test.d new file mode 100644 index 0000000..75b9333 --- /dev/null +++ b/tests/parser/lexer_test.d @@ -0,0 +1,67 @@ +/// Unit tests for the Antelope lexer. +/// +/// Tests tokenization of Makefile syntax: identifiers, operators, +/// variable references, comments, recipe lines, and line continuations. +module antelope.tests.parser.lexer_test; + +import antelope.parser.lexer; + +/// Test basic identifier and operator tokenization. +unittest +{ + auto lexer = Lexer("target: prereq\n"); + auto tok = lexer.nextToken(); + assert(tok.type == TokenType.identifier); + assert(tok.value == "target"); + + tok = lexer.nextToken(); + assert(tok.type == TokenType.colon); + + tok = lexer.nextToken(); + assert(tok.type == TokenType.identifier); + assert(tok.value == "prereq"); +} + +/// Test comment stripping — everything after '#' is ignored. +unittest +{ + auto lexer = Lexer("VAR = value # this is a comment\n"); + auto tok = lexer.nextToken(); + assert(tok.type == TokenType.identifier); + assert(tok.value == "VAR"); + + tok = lexer.nextToken(); + assert(tok.type == TokenType.equals); + + tok = lexer.nextToken(); + assert(tok.type == TokenType.identifier); + assert(tok.value == "value"); +} + +/// Test line continuation (backslash-newline). +unittest +{ + auto lexer = Lexer("target: prereq1 \\\n prereq2\n"); + auto tok = lexer.nextToken(); + assert(tok.type == TokenType.identifier); + assert(tok.value == "target"); + + tok = lexer.nextToken(); + assert(tok.type == TokenType.colon); +} + +/// Test variable reference prefix ($). +unittest +{ + auto lexer = Lexer("$(VAR)\n"); + auto tok = lexer.nextToken(); + assert(tok.type == TokenType.dollar); +} + +/// Test recipe line detection (tab-prefixed). +unittest +{ + auto lexer = Lexer("\tgcc -c hello.c\n"); + auto tok = lexer.nextToken(); + assert(tok.type == TokenType.recipeLine); +} diff --git a/tests/parser/parser_test.d b/tests/parser/parser_test.d new file mode 100644 index 0000000..aae2735 --- /dev/null +++ b/tests/parser/parser_test.d @@ -0,0 +1,30 @@ +/// Unit tests for the Antelope parser. +/// +/// Tests recursive-descent parsing of rules, variable assignments, +/// and directive blocks into the AST. +module antelope.tests.parser.parser_test; + +import antelope.parser.parser; +import antelope.parser.ast; + +/// Test parsing a simple rule with prerequisites and no recipe. +unittest +{ + auto root = parseMakefile("target: prereq1 prereq2\n"); + assert(root.type == AstType.rule_list); + assert(root.children.length == 1); + + auto rule = root.children[0]; + assert(rule.type == AstType.rule); +} + +/// Test parsing a variable assignment. +unittest +{ + auto root = parseMakefile("CC = gcc\n"); + assert(root.type == AstType.rule_list); + assert(root.children.length == 1); + + auto var = root.children[0]; + assert(var.type == AstType.variable_assignment); +}