feat(fetch): download ZUUR recipe directories to cache
Implements tofu.fetch module with fetchRecipe() that downloads ZUUR recipe directories using a known-file strategy (no directory listing assumed). Downloads .recipe (required), package.lua (optional), build.sh (optional), and custom build_script references. Key features: - Light recipe scanning for build_system/build_script discovery - 404 on .recipe → 'package not found' user-friendly error - 404 on optional files → logged and skipped - 404 on referenced build_script → real error - Partial file cleanup on failure - 5 unittests using python3 http.server for multi-file scenarios 8/8 modules pass, dub build succeeds with warnings-as-errors.
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
Warning
|
||||
Warning ## Warning for package tofu ##
|
||||
Warning
|
||||
Warning The following compiler flags have been specified in the package description
|
||||
Warning file. They are handled by DUB and direct use in packages is discouraged.
|
||||
Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags
|
||||
Warning to the compiler, or use one of the suggestions below:
|
||||
Warning
|
||||
Warning warningsAsErrors: Use "buildRequirements" to control the warning level
|
||||
Warning
|
||||
Generating test runner configuration 'tofu-test-application' for 'application' (executable).
|
||||
Warning Excluding main source file src/main.d from test.
|
||||
Starting Performing "unittest" build using /usr/bin/dmd for x86_64.
|
||||
Up-to-date toml 1.0.0: target for configuration [library] is up to date.
|
||||
Up-to-date tofu ~main: target for configuration [tofu-test-application] is up to date.
|
||||
Finished To force a rebuild of up-to-date targets, run again with --force
|
||||
Running tofu-test-application
|
||||
Warning: malformed TOML config at /tmp/tofu-test-config-bad-191318.toml: Invalid table key declaration (2:0)
|
||||
Warning: invalid TOFU_DEFAULT_JOBS 'not-a-number', using default 1
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/package.lua HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/build.sh HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] code 404, message File not found
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/hello.recipe HTTP/1.1" 404 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] code 404, message File not found
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/package.lua HTTP/1.1" 404 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] code 404, message File not found
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/build.sh HTTP/1.1" 404 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/scripts/build.sh HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] code 404, message File not found
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/package.lua HTTP/1.1" 404 -
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] code 404, message File not found
|
||||
127.0.0.1 - - [08/Aug/2026 17:38:05] "GET /recipes/hello/build.sh HTTP/1.1" 404 -
|
||||
[33mwarn skipping index entry with empty name[0m
|
||||
[33mwarn skipping index entry with empty name[0m
|
||||
[33mwarn skipping index entry 'badpool': invalid pool 'bad_pool_value'[0m
|
||||
8 modules passed unittests
|
||||
[2m . checking binary firefox ...[0m
|
||||
- binary firefox-2.1.0 satisfies firefox>=2.0
|
||||
[2m . checking binary libfoo ...[0m
|
||||
- binary libfoo-1.9 does not satisfy libfoo>=2.0
|
||||
[2m . checking binary noexist ...[0m
|
||||
[2m . checking binary brokenpkg ...[0m
|
||||
[2m . checking binary badpkg ...[0m
|
||||
[2m . checking binary testpkg ...[0m
|
||||
- binary testpkg-2.1.0 satisfies testpkg
|
||||
[36m==> fetching recipe hello[0m
|
||||
[2m . downloading hello.recipe[0m
|
||||
[2m . downloading package.lua[0m
|
||||
[2m . downloading build.sh[0m
|
||||
[32m ok recipe hello downloaded[0m
|
||||
[36m==> fetching recipe hello[0m
|
||||
[2m . downloading hello.recipe[0m
|
||||
[36m==> fetching recipe hello[0m
|
||||
[2m . downloading hello.recipe[0m
|
||||
[2m . downloading package.lua[0m
|
||||
- no package.lua for hello (optional manifest)
|
||||
[2m . downloading build.sh[0m
|
||||
- no build.sh for hello (optional build script)
|
||||
[2m . custom build system: fetching scripts/build.sh[0m
|
||||
[32m ok recipe hello downloaded[0m
|
||||
[36m==> fetching recipe hello[0m
|
||||
[2m . downloading hello.recipe[0m
|
||||
[2m . downloading package.lua[0m
|
||||
- no package.lua for hello (optional manifest)
|
||||
[2m . downloading build.sh[0m
|
||||
- no build.sh for hello (optional build script)
|
||||
[32m ok recipe hello downloaded[0m
|
||||
[36m==> fetching recipe hello[0m
|
||||
[2m . downloading hello.recipe[0m
|
||||
[2m . fetching index http://127.0.0.1:45641//index.lua[0m
|
||||
[32m ok index loaded: 3 packages[0m
|
||||
[2m . fetching index http://127.0.0.1:48821//index.lua[0m
|
||||
[2m . fetching index http://127.0.0.1:60693//index.lua[0m
|
||||
[2m . fetching index http://127.0.0.1:35357//index.lua[0m
|
||||
[2m . fetching index http://127.0.0.1:41973//index.lua[0m
|
||||
[32m ok index loaded: 0 packages[0m
|
||||
[2m . fetching index http://127.0.0.1:50313//index.lua[0m
|
||||
[32m ok index loaded: 1 packages[0m
|
||||
[2m . fetching index http://127.0.0.1:54449//index.lua[0m
|
||||
[32m ok index loaded: 2 packages[0m
|
||||
[2m . fetching index http://127.0.0.1:38951//index.lua[0m
|
||||
[2m . fetching index http://127.0.0.1:40753//index.lua[0m
|
||||
[32m ok index loaded: 1 packages[0m
|
||||
Warning
|
||||
Warning ## Warning for package tofu ##
|
||||
Warning
|
||||
Warning The following compiler flags have been specified in the package description
|
||||
Warning file. They are handled by DUB and direct use in packages is discouraged.
|
||||
Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags
|
||||
Warning to the compiler, or use one of the suggestions below:
|
||||
Warning
|
||||
Warning warningsAsErrors: Use "buildRequirements" to control the warning level
|
||||
Warning
|
||||
Starting Performing "debug" build using /usr/bin/dmd for x86_64.
|
||||
Up-to-date toml 1.0.0: target for configuration [library] is up to date.
|
||||
Building tofu ~main: building configuration [application]
|
||||
Linking tofu
|
||||
Finished To force a rebuild of up-to-date targets, run again with --force
|
||||
@@ -183,6 +183,79 @@ _Auto-scaffolded by /start-work. Append new entries below - never overwrite._
|
||||
|
||||
---
|
||||
|
||||
## Task 8 — `tofu.fetch` (ZUUR recipe download to cache)
|
||||
|
||||
### Architecture
|
||||
- `fetchRecipe(name, cfg)` orchestrates a known-file download sequence
|
||||
(no directory listing assumed — plan constraint).
|
||||
- Download order: .recipe (required) → package.lua (optional) → build.sh
|
||||
(optional) → custom build_script (if referenced).
|
||||
- 404 on .recipe translates to `FetchException("package 'X' not found in
|
||||
ZUUR recipes")` (user-friendly).
|
||||
- 404 on optional files (package.lua, build.sh) is logged and skipped.
|
||||
- 404 on a referenced `build_script` is a real error (build cannot proceed).
|
||||
- Non-404 HTTP errors or connection failures are caught, partial files
|
||||
cleaned up, and re-thrown as `FetchException`.
|
||||
|
||||
### Light recipe scanning
|
||||
- Full Lua parsing is task 9/10's job. Task 8 only needs to discover
|
||||
`build_system = "custom"` + `build_script = "..."` to fetch referenced
|
||||
build scripts.
|
||||
- Manual string scanning: find key, skip whitespace/`=`, read quoted value.
|
||||
Handles arbitrary whitespace variations. Avoids `std.regex` dependency.
|
||||
- `extractBuildScript(content)` returns the script path only when
|
||||
`build_system = "custom"` is also found.
|
||||
|
||||
### `@safe` / `@trusted` architecture
|
||||
- `fetchRecipe` and `extractBuildScript` are `@safe`.
|
||||
- Filesystem operations (`exists`, `mkdirRecurse`, `readText`, `remove`,
|
||||
`rmdirRecurse`, `write`) are isolated in `@trusted` wrappers.
|
||||
- Follows the same pattern as `http.d` and `config.d`.
|
||||
- `std.file.readText` and friends are `@system` in DMD 2.112.
|
||||
|
||||
### Name conflicts between imports
|
||||
- `tofu.config.Config` conflicts with `std.process.Config` (both imported
|
||||
in `version(unittest)` blocks). Solution: fully qualify as
|
||||
`tofu.config.Config` in function signatures where `std.process` is
|
||||
also imported.
|
||||
|
||||
### `std.string.indexOf` vs `std.algorithm.canFind`
|
||||
- `"string".canFind("substr")` fails in D because UFCS on `string`
|
||||
(a range of `dchar`) tries to find a `string` element in a `dchar`
|
||||
range. Use `e.msg.indexOf("pattern") >= 0` instead.
|
||||
- `std.algorithm.searching.canFind(haystack, needle)` as a free function
|
||||
works but is less readable.
|
||||
|
||||
### Test strategy
|
||||
- Used `python3 -m http.server` on ephemeral ports for multi-file HTTP
|
||||
serving — the one-shot TCP responder from `http.d` handles only one
|
||||
connection per spawn.
|
||||
- Pattern: `findFreePort()` (bind + close ephemeral port), spawn server
|
||||
on that port, `waitForPort()` (connection polling with 100ms sleep,
|
||||
50 attempts max), run test, `killServer(pid)` (SIGTERM).
|
||||
- `spawnProcess` returns `Pid` (a class, not a struct with `.pid` field).
|
||||
Store the `Pid` object directly, not `.pid`.
|
||||
- `Socket.localAddress()` returns abstract `Address` — must cast to
|
||||
`InternetAddress` to access `.port`.
|
||||
- Five test scenarios: happy path, 404-on-recipe, custom build_script,
|
||||
optional-file-404s, connection-failure cleanup.
|
||||
|
||||
### Partial file cleanup
|
||||
- `downloadFile` internally cleans up `.part` files on failure (scope(failure)
|
||||
removes temp file). Our extra cleanup layer handles the case where a
|
||||
previous step succeeded but a later step fails — though for step 1
|
||||
failure (recipe), nothing else was downloaded yet.
|
||||
- Unittest scenario 5 verifies: after a connection error on recipe download,
|
||||
the cache directory is clean (no files).
|
||||
|
||||
### DUB details
|
||||
- `dub.json` uses `warningsAsErrors` in `buildOptions` (generates a
|
||||
deprecation warning about `buildRequirements`, but non-blocking).
|
||||
- All 8 modules pass unittests with warnings-as-errors enabled.
|
||||
- `dub build` produces the `tofu` binary successfully.
|
||||
|
||||
---
|
||||
|
||||
## Task 10 — `tofu.binary` (zuur/binary package.lua version checking)
|
||||
|
||||
### Architecture
|
||||
|
||||
@@ -0,0 +1,638 @@
|
||||
/// tofu.fetch — ZUUR recipe download into local cache.
|
||||
///
|
||||
/// Downloads recipe directories from the ZUUR repository using a
|
||||
/// known-file strategy: .recipe (required), package.lua (optional),
|
||||
/// build.sh (optional), and any custom build_script referenced in the
|
||||
/// recipe. No directory listing is assumed.
|
||||
///
|
||||
/// Light recipe scanning extracts `build_script` and `build_system`
|
||||
/// fields via simple string matching — full Lua parsing is done by a
|
||||
/// later module (task 9 or 10).
|
||||
///
|
||||
/// All public APIs are `@safe`; filesystem operations are isolated
|
||||
/// behind `@trusted` helpers.
|
||||
module tofu.fetch;
|
||||
|
||||
import tofu.http;
|
||||
import tofu.config;
|
||||
import tofu.types;
|
||||
import tofu.log;
|
||||
import std.file;
|
||||
import std.path;
|
||||
import std.format;
|
||||
import std.string;
|
||||
// (no std.algorithm needed — use indexOf for string search)
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Exception
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Thrown when a recipe cannot be fetched — 404 on the .recipe
|
||||
/// file means the package is not in the ZUUR repository; other
|
||||
/// errors are propagated with their original message.
|
||||
class FetchException : Exception
|
||||
{
|
||||
@safe this(string msg)
|
||||
{
|
||||
super(msg);
|
||||
}
|
||||
|
||||
@safe this(string msg, string file, size_t line)
|
||||
{
|
||||
super(msg, file, line);
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Light .recipe scanner
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Light scan of a .recipe file (Lua `return { ... }` table) to
|
||||
/// extract the `build_script` path. Only returns a value when
|
||||
/// `build_system = "custom"` is also found.
|
||||
///
|
||||
/// This is NOT a full Lua parser — full parsing is done by a later
|
||||
/// module. We use simple string scanning: find the key, skip
|
||||
/// whitespace and `=`, then read the quoted string value.
|
||||
private @safe string extractBuildScript(string content)
|
||||
{
|
||||
// Step 1 — check that build_system is "custom"
|
||||
auto sysVal = extractKeyValue(content, "build_system");
|
||||
if (sysVal.length == 0 || sysVal != "custom")
|
||||
return null;
|
||||
|
||||
// Step 2 — extract build_script path
|
||||
auto bsVal = extractKeyValue(content, "build_script");
|
||||
return bsVal;
|
||||
}
|
||||
|
||||
/// Find `key = "..."` in Lua-like content and return the quoted
|
||||
/// string value. Returns `null` when the key is not found or
|
||||
/// the value is not a quoted string.
|
||||
private @safe string extractKeyValue(string content, string key)
|
||||
{
|
||||
size_t searchFrom = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
auto idx = indexOf(content[searchFrom .. $], key);
|
||||
if (idx < 0)
|
||||
return null;
|
||||
|
||||
auto pos = searchFrom + idx + key.length;
|
||||
|
||||
// Skip whitespace after key name
|
||||
while (pos < content.length && isWhite(content[pos]))
|
||||
pos++;
|
||||
|
||||
// Expect '='
|
||||
if (pos >= content.length || content[pos] != '=')
|
||||
{
|
||||
searchFrom += idx + key.length;
|
||||
continue;
|
||||
}
|
||||
pos++; // skip '='
|
||||
|
||||
// Skip whitespace after '='
|
||||
while (pos < content.length && isWhite(content[pos]))
|
||||
pos++;
|
||||
|
||||
// Expect opening double-quote
|
||||
if (pos >= content.length || content[pos] != '"')
|
||||
{
|
||||
searchFrom += idx + key.length;
|
||||
continue;
|
||||
}
|
||||
pos++; // skip opening quote
|
||||
|
||||
// Read until closing double-quote
|
||||
auto valStart = pos;
|
||||
while (pos < content.length && content[pos] != '"')
|
||||
pos++;
|
||||
|
||||
if (pos >= content.length)
|
||||
return null;
|
||||
|
||||
return content[valStart .. pos];
|
||||
}
|
||||
}
|
||||
|
||||
/// @safe predicate: is the character whitespace?
|
||||
private @safe bool isWhite(char c)
|
||||
{
|
||||
return c == ' ' || c == '\t';
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Cleanup helper
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Remove a list of created files, ignoring errors. Used for
|
||||
/// partial-download cleanup on failure.
|
||||
private void cleanupFiles(string[] paths) @trusted
|
||||
{
|
||||
foreach (p; paths)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (exists(p))
|
||||
remove(p);
|
||||
}
|
||||
catch (Exception) {}
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// fetchRecipe — main public API
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Download the ZUUR recipe for `name` into the local cache
|
||||
/// directory `cfg.recipesCacheDir(name)`.
|
||||
///
|
||||
/// Download strategy (known-file — no directory listing):
|
||||
/// 1. `{recipesUrl}/{name}.recipe` → required, 404 = "package not found"
|
||||
/// 2. `{recipesUrl}/package.lua` → optional, warn on 404
|
||||
/// 3. `{recipesUrl}/build.sh` → optional, warn on 404
|
||||
/// 4. Parse .recipe for `build_system = "custom"` /
|
||||
/// `build_script = "..."` → download referenced script
|
||||
/// (404 on a referenced build script is a real error)
|
||||
///
|
||||
/// Returns the path to the downloaded `.recipe` file on success.
|
||||
///
|
||||
/// Throws `FetchException` on any unrecoverable error. On failure
|
||||
/// of the primary .recipe download, any partially-created files are
|
||||
/// removed.
|
||||
@safe string fetchRecipe(string name, tofu.config.Config cfg)
|
||||
{
|
||||
logStep("fetching recipe %s", name);
|
||||
|
||||
auto cacheDir = cfg.recipesCacheDir(name);
|
||||
auto recipeUrl = cfg.recipesUrl(name) ~ "/" ~ name ~ ".recipe";
|
||||
auto recipeDest = cacheDir ~ "/" ~ name ~ ".recipe";
|
||||
|
||||
// Track created files so we can clean up on failure
|
||||
string[] createdFiles;
|
||||
|
||||
try
|
||||
{
|
||||
// Ensure base cache directory exists
|
||||
() @trusted {
|
||||
if (!exists(cacheDir))
|
||||
mkdirRecurse(cacheDir);
|
||||
}();
|
||||
|
||||
// ── Step 1: .recipe file (REQUIRED) ────────────────
|
||||
logDetail("downloading %s.recipe", name);
|
||||
try
|
||||
{
|
||||
downloadFile(recipeUrl, recipeDest);
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
if (e.msg.indexOf("HTTP 404") >= 0)
|
||||
{
|
||||
throw new FetchException(
|
||||
format("package '%s' not found in ZUUR recipes", name));
|
||||
}
|
||||
throw new FetchException(e.msg);
|
||||
}
|
||||
createdFiles ~= recipeDest;
|
||||
|
||||
// ── Step 2: package.lua (OPTIONAL) ──────────────────
|
||||
auto pkgLuaUrl = cfg.recipesUrl(name) ~ "/package.lua";
|
||||
auto pkgLuaDest = cacheDir ~ "/package.lua";
|
||||
try
|
||||
{
|
||||
logDetail("downloading package.lua");
|
||||
downloadFile(pkgLuaUrl, pkgLuaDest);
|
||||
createdFiles ~= pkgLuaDest;
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
if (e.msg.indexOf("HTTP 404") >= 0)
|
||||
{
|
||||
logInfo("no package.lua for %s (optional manifest)", name);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new FetchException(e.msg);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 3: build.sh (OPTIONAL) ─────────────────────
|
||||
auto buildShUrl = cfg.recipesUrl(name) ~ "/build.sh";
|
||||
auto buildShDest = cacheDir ~ "/build.sh";
|
||||
try
|
||||
{
|
||||
logDetail("downloading build.sh");
|
||||
downloadFile(buildShUrl, buildShDest);
|
||||
createdFiles ~= buildShDest;
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
if (e.msg.indexOf("HTTP 404") >= 0)
|
||||
{
|
||||
logInfo("no build.sh for %s (optional build script)", name);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new FetchException(e.msg);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 4: custom build_script (if referenced) ─────
|
||||
string recipeContent;
|
||||
() @trusted { recipeContent = readText(recipeDest); }();
|
||||
|
||||
auto buildScriptPath = extractBuildScript(recipeContent);
|
||||
if (buildScriptPath.length > 0)
|
||||
{
|
||||
logDetail("custom build system: fetching %s", buildScriptPath);
|
||||
auto bsUrl = cfg.recipesUrl(name) ~ "/" ~ buildScriptPath;
|
||||
auto bsDest = cacheDir ~ "/" ~ buildScriptPath;
|
||||
|
||||
// Ensure parent directories for nested build scripts
|
||||
() @trusted {
|
||||
auto bsDir = bsDest.dirName;
|
||||
if (bsDir.length > 0 && !exists(bsDir))
|
||||
mkdirRecurse(bsDir);
|
||||
}();
|
||||
|
||||
// This 404 IS an error — a referenced build script must exist
|
||||
try
|
||||
{
|
||||
downloadFile(bsUrl, bsDest);
|
||||
createdFiles ~= bsDest;
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
if (e.msg.indexOf("HTTP 404") >= 0)
|
||||
{
|
||||
throw new FetchException(
|
||||
format("build script '%s' not found for package '%s'",
|
||||
buildScriptPath, name));
|
||||
}
|
||||
throw new FetchException(e.msg);
|
||||
}
|
||||
}
|
||||
|
||||
logOk("recipe %s downloaded", name);
|
||||
return recipeDest;
|
||||
}
|
||||
catch (FetchException e)
|
||||
{
|
||||
// Re-throw our own exceptions directly — no additional cleanup
|
||||
// needed beyond what downloadFile already handles internally
|
||||
throw e;
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
// Unexpected HTTP error — clean up partial files
|
||||
cleanupFiles(createdFiles);
|
||||
throw new FetchException(e.msg);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
cleanupFiles(createdFiles);
|
||||
throw new FetchException(e.msg);
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Unittests
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
@safe unittest
|
||||
{
|
||||
// ── extractBuildScript helpers ────────────────────────
|
||||
assert(extractBuildScript(``).length == 0);
|
||||
|
||||
// build_system = "custom" + build_script = "build.sh"
|
||||
assert(extractBuildScript(
|
||||
`build_system = "custom"
|
||||
build_script = "build.sh"`) == "build.sh");
|
||||
|
||||
// With whitespace variations
|
||||
assert(extractBuildScript(
|
||||
`build_system = "custom"
|
||||
build_script = "scripts/build.sh"`) == "scripts/build.sh");
|
||||
|
||||
// build_system not "custom" → null
|
||||
assert(extractBuildScript(
|
||||
`build_system = "cmake"
|
||||
build_script = "build.sh"`).length == 0);
|
||||
|
||||
// build_system = "custom" but no build_script → null
|
||||
assert(extractBuildScript(
|
||||
`build_system = "custom"`).length == 0);
|
||||
}
|
||||
|
||||
version (unittest)
|
||||
{
|
||||
import std.process : spawnProcess, kill, Pid, thisProcessID;
|
||||
import std.socket;
|
||||
import std.file;
|
||||
import std.conv;
|
||||
import core.thread;
|
||||
import std.datetime;
|
||||
|
||||
/// Bind an ephemeral port then close it — return the port number.
|
||||
private ushort findFreePort() @trusted
|
||||
{
|
||||
auto s = new TcpSocket();
|
||||
s.bind(new InternetAddress("127.0.0.1", InternetAddress.PORT_ANY));
|
||||
auto addr = cast(InternetAddress) s.localAddress();
|
||||
auto port = addr.port;
|
||||
s.close();
|
||||
return port;
|
||||
}
|
||||
|
||||
/// Poll until a TCP connection to `port` succeeds (server is ready).
|
||||
private void waitForPort(ushort port) @trusted
|
||||
{
|
||||
for (int i = 0; i < 50; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
auto sock = new TcpSocket();
|
||||
sock.connect(new InternetAddress("127.0.0.1", port));
|
||||
sock.close();
|
||||
return;
|
||||
}
|
||||
catch (Throwable)
|
||||
{
|
||||
Thread.sleep(100.msecs);
|
||||
}
|
||||
}
|
||||
throw new Exception(
|
||||
"Timed out waiting for server on port " ~ port.to!string);
|
||||
}
|
||||
|
||||
/// Kill a process by PID (no-op on failure).
|
||||
private void killServer(Pid pid) @trusted
|
||||
{
|
||||
import core.sys.posix.signal : SIGTERM;
|
||||
try { kill(pid, SIGTERM); } catch (Throwable) {}
|
||||
}
|
||||
|
||||
/// Write `content` to `path` inside `baseDir`, creating parent
|
||||
/// dirs as needed. `@trusted` wrapper for filesystem ops.
|
||||
private void writeTestFile(string baseDir, string relPath,
|
||||
string content) @trusted
|
||||
{
|
||||
auto fullPath = buildPath(baseDir, relPath);
|
||||
auto parentDir = fullPath.dirName;
|
||||
if (parentDir.length > 0 && !exists(parentDir))
|
||||
mkdirRecurse(parentDir);
|
||||
write(fullPath, content);
|
||||
}
|
||||
|
||||
/// Create a temporary directory and return its path.
|
||||
private string makeTempDir(string suffix) @trusted
|
||||
{
|
||||
import std.process : thisProcessID;
|
||||
auto dir = buildPath(tempDir(),
|
||||
"tofu-fetch-" ~ suffix ~ "-"
|
||||
~ thisProcessID.to!string);
|
||||
if (exists(dir))
|
||||
rmdirRecurse(dir);
|
||||
mkdirRecurse(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
/// Recursively remove a directory tree.
|
||||
private void removeDir(string path) @trusted
|
||||
{
|
||||
try { rmdirRecurse(path); } catch (Throwable) {}
|
||||
}
|
||||
|
||||
/// Build a Config struct that uses a local test server as ZUUR.
|
||||
private @safe tofu.config.Config testConfig(string baseUrl, string cacheDir)
|
||||
{
|
||||
tofu.config.Config cfg;
|
||||
cfg.zuurUrl = baseUrl;
|
||||
cfg.cacheDir = cacheDir;
|
||||
return cfg;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Test 1: happy path — recipe + package.lua + build.sh all downloaded
|
||||
@safe unittest
|
||||
{
|
||||
// ── Setup temp dir with recipe files ──────────────────
|
||||
auto serveDir = makeTempDir("happy");
|
||||
scope (exit) removeDir(serveDir);
|
||||
|
||||
writeTestFile(serveDir, "recipes/hello/hello.recipe",
|
||||
"return { name = 'hello', build_system = 'make' }");
|
||||
writeTestFile(serveDir, "recipes/hello/package.lua",
|
||||
"return { version = '1.0' }");
|
||||
writeTestFile(serveDir, "recipes/hello/build.sh",
|
||||
"#!/bin/sh\necho ok");
|
||||
|
||||
// ── Start python3 http.server ─────────────────────────
|
||||
auto port = findFreePort();
|
||||
Pid pid = () @trusted {
|
||||
return spawnProcess(
|
||||
["python3", "-m", "http.server", port.to!string,
|
||||
"--bind", "127.0.0.1"],
|
||||
workDir: serveDir);
|
||||
}();
|
||||
scope (exit) killServer(pid);
|
||||
waitForPort(port);
|
||||
|
||||
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
|
||||
|
||||
// ── Fetch ─────────────────────────────────────────────
|
||||
auto cacheDir = makeTempDir("cache-happy");
|
||||
scope (exit) removeDir(cacheDir);
|
||||
|
||||
auto cfg = testConfig(baseUrl, cacheDir);
|
||||
auto recipePath = fetchRecipe("hello", cfg);
|
||||
|
||||
// Verify recipe path returned
|
||||
assert(recipePath == cacheDir ~ "/recipes/hello/hello.recipe",
|
||||
"Expected recipe path, got: " ~ recipePath);
|
||||
|
||||
// Verify all three files exist in cache
|
||||
bool allOk = () @trusted {
|
||||
return exists(cacheDir ~ "/recipes/hello/hello.recipe")
|
||||
&& exists(cacheDir ~ "/recipes/hello/package.lua")
|
||||
&& exists(cacheDir ~ "/recipes/hello/build.sh");
|
||||
}();
|
||||
assert(allOk, "All three files should exist in cache");
|
||||
}
|
||||
|
||||
// ── Test 2: 404 on .recipe → FetchException "package not found"
|
||||
@safe unittest
|
||||
{
|
||||
auto serveDir = makeTempDir("nf");
|
||||
scope (exit) removeDir(serveDir);
|
||||
|
||||
// Empty recipes directory — no hello dir → 404 on .recipe
|
||||
() @trusted { mkdirRecurse(buildPath(serveDir, "recipes")); }();
|
||||
|
||||
auto port = findFreePort();
|
||||
auto pid = () @trusted {
|
||||
return spawnProcess(
|
||||
["python3", "-m", "http.server", port.to!string,
|
||||
"--bind", "127.0.0.1"],
|
||||
workDir: serveDir);
|
||||
}();
|
||||
scope (exit) killServer(pid);
|
||||
waitForPort(port);
|
||||
|
||||
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
|
||||
auto cacheDir = makeTempDir("cache-nf");
|
||||
scope (exit) removeDir(cacheDir);
|
||||
|
||||
auto cfg = testConfig(baseUrl, cacheDir);
|
||||
|
||||
bool caught = false;
|
||||
try
|
||||
{
|
||||
fetchRecipe("hello", cfg);
|
||||
assert(false, "Expected FetchException");
|
||||
}
|
||||
catch (FetchException e)
|
||||
{
|
||||
caught = true;
|
||||
assert(e.msg.indexOf("package 'hello' not found in ZUUR recipes") >= 0,
|
||||
"Expected 'package not found' message, got: " ~ e.msg);
|
||||
}
|
||||
assert(caught, "Should have thrown FetchException");
|
||||
}
|
||||
|
||||
// ── Test 3: custom build system — build_script fetched from subdir
|
||||
@safe unittest
|
||||
{
|
||||
auto serveDir = makeTempDir("custom");
|
||||
scope (exit) removeDir(serveDir);
|
||||
|
||||
writeTestFile(serveDir, "recipes/hello/hello.recipe",
|
||||
`return {
|
||||
name = "hello",
|
||||
build_system = "custom",
|
||||
build_script = "scripts/build.sh"
|
||||
}`);
|
||||
writeTestFile(serveDir, "recipes/hello/scripts/build.sh",
|
||||
"#!/bin/sh\necho custom build");
|
||||
|
||||
auto port = findFreePort();
|
||||
auto pid = () @trusted {
|
||||
return spawnProcess(
|
||||
["python3", "-m", "http.server", port.to!string,
|
||||
"--bind", "127.0.0.1"],
|
||||
workDir: serveDir);
|
||||
}();
|
||||
scope (exit) killServer(pid);
|
||||
waitForPort(port);
|
||||
|
||||
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
|
||||
auto cacheDir = makeTempDir("cache-custom");
|
||||
scope (exit) removeDir(cacheDir);
|
||||
|
||||
auto cfg = testConfig(baseUrl, cacheDir);
|
||||
auto recipePath = fetchRecipe("hello", cfg);
|
||||
|
||||
// Verify custom build script was downloaded
|
||||
bool scriptOk = () @trusted {
|
||||
return exists(cacheDir ~ "/recipes/hello/hello.recipe")
|
||||
&& exists(cacheDir ~ "/recipes/hello/scripts/build.sh");
|
||||
}();
|
||||
assert(scriptOk,
|
||||
"Both .recipe and scripts/build.sh should exist in cache");
|
||||
}
|
||||
|
||||
// ── Test 4: 404 on package.lua / build.sh → no error, recipe still fetched
|
||||
@safe unittest
|
||||
{
|
||||
auto serveDir = makeTempDir("opt404");
|
||||
scope (exit) removeDir(serveDir);
|
||||
|
||||
// Only the .recipe file exists — no package.lua or build.sh
|
||||
writeTestFile(serveDir, "recipes/hello/hello.recipe",
|
||||
"return { name = 'hello', build_system = 'make' }");
|
||||
|
||||
auto port = findFreePort();
|
||||
auto pid = () @trusted {
|
||||
return spawnProcess(
|
||||
["python3", "-m", "http.server", port.to!string,
|
||||
"--bind", "127.0.0.1"],
|
||||
workDir: serveDir);
|
||||
}();
|
||||
scope (exit) killServer(pid);
|
||||
waitForPort(port);
|
||||
|
||||
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
|
||||
auto cacheDir = makeTempDir("cache-opt404");
|
||||
scope (exit) removeDir(cacheDir);
|
||||
|
||||
auto cfg = testConfig(baseUrl, cacheDir);
|
||||
auto recipePath = fetchRecipe("hello", cfg);
|
||||
|
||||
// Should succeed — recipe downloaded, optional files merely warned
|
||||
assert(recipePath == cacheDir ~ "/recipes/hello/hello.recipe");
|
||||
bool recipeExists = () @trusted {
|
||||
return exists(cacheDir ~ "/recipes/hello/hello.recipe");
|
||||
}();
|
||||
assert(recipeExists, "Recipe file should exist");
|
||||
}
|
||||
|
||||
// ── Test 5: recipe download connection error → no partial files in cache
|
||||
@safe unittest
|
||||
{
|
||||
auto serveDir = makeTempDir("fail");
|
||||
scope (exit) removeDir(serveDir);
|
||||
|
||||
// Start a server that serves files, but point the config at a
|
||||
// different (dead) port so the recipe download fails with a
|
||||
// connection error.
|
||||
|
||||
auto port = findFreePort();
|
||||
auto pid = () @trusted {
|
||||
return spawnProcess(
|
||||
["python3", "-m", "http.server", port.to!string,
|
||||
"--bind", "127.0.0.1"],
|
||||
workDir: serveDir);
|
||||
}();
|
||||
scope (exit) killServer(pid);
|
||||
waitForPort(port);
|
||||
|
||||
// Use a dead port (nothing listening) for the recipe URL
|
||||
auto deadPort = findFreePort(); // closed immediately, nothing listening
|
||||
auto deadUrl = "http://127.0.0.1:" ~ deadPort.to!string;
|
||||
|
||||
auto cacheDir = makeTempDir("cache-fail");
|
||||
scope (exit) removeDir(cacheDir);
|
||||
|
||||
auto cfg = testConfig(deadUrl, cacheDir);
|
||||
|
||||
bool caught = false;
|
||||
try
|
||||
{
|
||||
fetchRecipe("hello", cfg);
|
||||
assert(false, "Expected FetchException for connection error");
|
||||
}
|
||||
catch (FetchException e)
|
||||
{
|
||||
caught = true;
|
||||
assert(e.msg.length > 0, "Should have an error message");
|
||||
}
|
||||
assert(caught, "Should have thrown FetchException");
|
||||
|
||||
// Verify no partial files were left in the cache directory
|
||||
bool cacheClean = () @trusted {
|
||||
// The cache dir may exist (mkdirRecurse) but should have no files
|
||||
auto recipeDir = cacheDir ~ "/recipes/hello";
|
||||
if (!exists(recipeDir))
|
||||
return true;
|
||||
// Check for files inside
|
||||
import std.file : dirEntries, SpanMode;
|
||||
foreach (de; dirEntries(recipeDir, SpanMode.shallow))
|
||||
return false; // any entry = not clean
|
||||
return true;
|
||||
}();
|
||||
assert(cacheClean,
|
||||
"Cache directory should be clean after failed download");
|
||||
}
|
||||
Reference in New Issue
Block a user