feat(fetch): download ZUUR recipe directories to cache

Implements tofu.fetch module with fetchRecipe() that downloads
ZUUR recipe directories using a known-file strategy (no directory
listing assumed). Downloads .recipe (required), package.lua (optional),
build.sh (optional), and custom build_script references.

Key features:
- Light recipe scanning for build_system/build_script discovery
- 404 on .recipe → 'package not found' user-friendly error
- 404 on optional files → logged and skipped
- 404 on referenced build_script → real error
- Partial file cleanup on failure
- 5 unittests using python3 http.server for multi-file scenarios

8/8 modules pass, dub build succeeds with warnings-as-errors.
This commit is contained in:
2026-08-08 17:38:41 -04:00
parent b367f21d29
commit d7d0d2ec6d
3 changed files with 812 additions and 0 deletions
+638
View File
@@ -0,0 +1,638 @@
/// tofu.fetch — ZUUR recipe download into local cache.
///
/// Downloads recipe directories from the ZUUR repository using a
/// known-file strategy: .recipe (required), package.lua (optional),
/// build.sh (optional), and any custom build_script referenced in the
/// recipe. No directory listing is assumed.
///
/// Light recipe scanning extracts `build_script` and `build_system`
/// fields via simple string matching — full Lua parsing is done by a
/// later module (task 9 or 10).
///
/// All public APIs are `@safe`; filesystem operations are isolated
/// behind `@trusted` helpers.
module tofu.fetch;
import tofu.http;
import tofu.config;
import tofu.types;
import tofu.log;
import std.file;
import std.path;
import std.format;
import std.string;
// (no std.algorithm needed — use indexOf for string search)
// ────────────────────────────────────────────────────────────
// Exception
// ────────────────────────────────────────────────────────────
/// Thrown when a recipe cannot be fetched — 404 on the .recipe
/// file means the package is not in the ZUUR repository; other
/// errors are propagated with their original message.
class FetchException : Exception
{
@safe this(string msg)
{
super(msg);
}
@safe this(string msg, string file, size_t line)
{
super(msg, file, line);
}
}
// ────────────────────────────────────────────────────────────
// Light .recipe scanner
// ────────────────────────────────────────────────────────────
/// Light scan of a .recipe file (Lua `return { ... }` table) to
/// extract the `build_script` path. Only returns a value when
/// `build_system = "custom"` is also found.
///
/// This is NOT a full Lua parser — full parsing is done by a later
/// module. We use simple string scanning: find the key, skip
/// whitespace and `=`, then read the quoted string value.
private @safe string extractBuildScript(string content)
{
// Step 1 — check that build_system is "custom"
auto sysVal = extractKeyValue(content, "build_system");
if (sysVal.length == 0 || sysVal != "custom")
return null;
// Step 2 — extract build_script path
auto bsVal = extractKeyValue(content, "build_script");
return bsVal;
}
/// Find `key = "..."` in Lua-like content and return the quoted
/// string value. Returns `null` when the key is not found or
/// the value is not a quoted string.
private @safe string extractKeyValue(string content, string key)
{
size_t searchFrom = 0;
while (true)
{
auto idx = indexOf(content[searchFrom .. $], key);
if (idx < 0)
return null;
auto pos = searchFrom + idx + key.length;
// Skip whitespace after key name
while (pos < content.length && isWhite(content[pos]))
pos++;
// Expect '='
if (pos >= content.length || content[pos] != '=')
{
searchFrom += idx + key.length;
continue;
}
pos++; // skip '='
// Skip whitespace after '='
while (pos < content.length && isWhite(content[pos]))
pos++;
// Expect opening double-quote
if (pos >= content.length || content[pos] != '"')
{
searchFrom += idx + key.length;
continue;
}
pos++; // skip opening quote
// Read until closing double-quote
auto valStart = pos;
while (pos < content.length && content[pos] != '"')
pos++;
if (pos >= content.length)
return null;
return content[valStart .. pos];
}
}
/// @safe predicate: is the character whitespace?
private @safe bool isWhite(char c)
{
return c == ' ' || c == '\t';
}
// ────────────────────────────────────────────────────────────
// Cleanup helper
// ────────────────────────────────────────────────────────────
/// Remove a list of created files, ignoring errors. Used for
/// partial-download cleanup on failure.
private void cleanupFiles(string[] paths) @trusted
{
foreach (p; paths)
{
try
{
if (exists(p))
remove(p);
}
catch (Exception) {}
}
}
// ────────────────────────────────────────────────────────────
// fetchRecipe — main public API
// ────────────────────────────────────────────────────────────
/// Download the ZUUR recipe for `name` into the local cache
/// directory `cfg.recipesCacheDir(name)`.
///
/// Download strategy (known-file — no directory listing):
/// 1. `{recipesUrl}/{name}.recipe` → required, 404 = "package not found"
/// 2. `{recipesUrl}/package.lua` → optional, warn on 404
/// 3. `{recipesUrl}/build.sh` → optional, warn on 404
/// 4. Parse .recipe for `build_system = "custom"` /
/// `build_script = "..."` → download referenced script
/// (404 on a referenced build script is a real error)
///
/// Returns the path to the downloaded `.recipe` file on success.
///
/// Throws `FetchException` on any unrecoverable error. On failure
/// of the primary .recipe download, any partially-created files are
/// removed.
@safe string fetchRecipe(string name, tofu.config.Config cfg)
{
logStep("fetching recipe %s", name);
auto cacheDir = cfg.recipesCacheDir(name);
auto recipeUrl = cfg.recipesUrl(name) ~ "/" ~ name ~ ".recipe";
auto recipeDest = cacheDir ~ "/" ~ name ~ ".recipe";
// Track created files so we can clean up on failure
string[] createdFiles;
try
{
// Ensure base cache directory exists
() @trusted {
if (!exists(cacheDir))
mkdirRecurse(cacheDir);
}();
// ── Step 1: .recipe file (REQUIRED) ────────────────
logDetail("downloading %s.recipe", name);
try
{
downloadFile(recipeUrl, recipeDest);
}
catch (HttpException e)
{
if (e.msg.indexOf("HTTP 404") >= 0)
{
throw new FetchException(
format("package '%s' not found in ZUUR recipes", name));
}
throw new FetchException(e.msg);
}
createdFiles ~= recipeDest;
// ── Step 2: package.lua (OPTIONAL) ──────────────────
auto pkgLuaUrl = cfg.recipesUrl(name) ~ "/package.lua";
auto pkgLuaDest = cacheDir ~ "/package.lua";
try
{
logDetail("downloading package.lua");
downloadFile(pkgLuaUrl, pkgLuaDest);
createdFiles ~= pkgLuaDest;
}
catch (HttpException e)
{
if (e.msg.indexOf("HTTP 404") >= 0)
{
logInfo("no package.lua for %s (optional manifest)", name);
}
else
{
throw new FetchException(e.msg);
}
}
// ── Step 3: build.sh (OPTIONAL) ─────────────────────
auto buildShUrl = cfg.recipesUrl(name) ~ "/build.sh";
auto buildShDest = cacheDir ~ "/build.sh";
try
{
logDetail("downloading build.sh");
downloadFile(buildShUrl, buildShDest);
createdFiles ~= buildShDest;
}
catch (HttpException e)
{
if (e.msg.indexOf("HTTP 404") >= 0)
{
logInfo("no build.sh for %s (optional build script)", name);
}
else
{
throw new FetchException(e.msg);
}
}
// ── Step 4: custom build_script (if referenced) ─────
string recipeContent;
() @trusted { recipeContent = readText(recipeDest); }();
auto buildScriptPath = extractBuildScript(recipeContent);
if (buildScriptPath.length > 0)
{
logDetail("custom build system: fetching %s", buildScriptPath);
auto bsUrl = cfg.recipesUrl(name) ~ "/" ~ buildScriptPath;
auto bsDest = cacheDir ~ "/" ~ buildScriptPath;
// Ensure parent directories for nested build scripts
() @trusted {
auto bsDir = bsDest.dirName;
if (bsDir.length > 0 && !exists(bsDir))
mkdirRecurse(bsDir);
}();
// This 404 IS an error — a referenced build script must exist
try
{
downloadFile(bsUrl, bsDest);
createdFiles ~= bsDest;
}
catch (HttpException e)
{
if (e.msg.indexOf("HTTP 404") >= 0)
{
throw new FetchException(
format("build script '%s' not found for package '%s'",
buildScriptPath, name));
}
throw new FetchException(e.msg);
}
}
logOk("recipe %s downloaded", name);
return recipeDest;
}
catch (FetchException e)
{
// Re-throw our own exceptions directly — no additional cleanup
// needed beyond what downloadFile already handles internally
throw e;
}
catch (HttpException e)
{
// Unexpected HTTP error — clean up partial files
cleanupFiles(createdFiles);
throw new FetchException(e.msg);
}
catch (Exception e)
{
cleanupFiles(createdFiles);
throw new FetchException(e.msg);
}
}
// ────────────────────────────────────────────────────────────
// Unittests
// ────────────────────────────────────────────────────────────
@safe unittest
{
// ── extractBuildScript helpers ────────────────────────
assert(extractBuildScript(``).length == 0);
// build_system = "custom" + build_script = "build.sh"
assert(extractBuildScript(
`build_system = "custom"
build_script = "build.sh"`) == "build.sh");
// With whitespace variations
assert(extractBuildScript(
`build_system = "custom"
build_script = "scripts/build.sh"`) == "scripts/build.sh");
// build_system not "custom" → null
assert(extractBuildScript(
`build_system = "cmake"
build_script = "build.sh"`).length == 0);
// build_system = "custom" but no build_script → null
assert(extractBuildScript(
`build_system = "custom"`).length == 0);
}
version (unittest)
{
import std.process : spawnProcess, kill, Pid, thisProcessID;
import std.socket;
import std.file;
import std.conv;
import core.thread;
import std.datetime;
/// Bind an ephemeral port then close it — return the port number.
private ushort findFreePort() @trusted
{
auto s = new TcpSocket();
s.bind(new InternetAddress("127.0.0.1", InternetAddress.PORT_ANY));
auto addr = cast(InternetAddress) s.localAddress();
auto port = addr.port;
s.close();
return port;
}
/// Poll until a TCP connection to `port` succeeds (server is ready).
private void waitForPort(ushort port) @trusted
{
for (int i = 0; i < 50; i++)
{
try
{
auto sock = new TcpSocket();
sock.connect(new InternetAddress("127.0.0.1", port));
sock.close();
return;
}
catch (Throwable)
{
Thread.sleep(100.msecs);
}
}
throw new Exception(
"Timed out waiting for server on port " ~ port.to!string);
}
/// Kill a process by PID (no-op on failure).
private void killServer(Pid pid) @trusted
{
import core.sys.posix.signal : SIGTERM;
try { kill(pid, SIGTERM); } catch (Throwable) {}
}
/// Write `content` to `path` inside `baseDir`, creating parent
/// dirs as needed. `@trusted` wrapper for filesystem ops.
private void writeTestFile(string baseDir, string relPath,
string content) @trusted
{
auto fullPath = buildPath(baseDir, relPath);
auto parentDir = fullPath.dirName;
if (parentDir.length > 0 && !exists(parentDir))
mkdirRecurse(parentDir);
write(fullPath, content);
}
/// Create a temporary directory and return its path.
private string makeTempDir(string suffix) @trusted
{
import std.process : thisProcessID;
auto dir = buildPath(tempDir(),
"tofu-fetch-" ~ suffix ~ "-"
~ thisProcessID.to!string);
if (exists(dir))
rmdirRecurse(dir);
mkdirRecurse(dir);
return dir;
}
/// Recursively remove a directory tree.
private void removeDir(string path) @trusted
{
try { rmdirRecurse(path); } catch (Throwable) {}
}
/// Build a Config struct that uses a local test server as ZUUR.
private @safe tofu.config.Config testConfig(string baseUrl, string cacheDir)
{
tofu.config.Config cfg;
cfg.zuurUrl = baseUrl;
cfg.cacheDir = cacheDir;
return cfg;
}
}
// ── Test 1: happy path — recipe + package.lua + build.sh all downloaded
@safe unittest
{
// ── Setup temp dir with recipe files ──────────────────
auto serveDir = makeTempDir("happy");
scope (exit) removeDir(serveDir);
writeTestFile(serveDir, "recipes/hello/hello.recipe",
"return { name = 'hello', build_system = 'make' }");
writeTestFile(serveDir, "recipes/hello/package.lua",
"return { version = '1.0' }");
writeTestFile(serveDir, "recipes/hello/build.sh",
"#!/bin/sh\necho ok");
// ── Start python3 http.server ─────────────────────────
auto port = findFreePort();
Pid pid = () @trusted {
return spawnProcess(
["python3", "-m", "http.server", port.to!string,
"--bind", "127.0.0.1"],
workDir: serveDir);
}();
scope (exit) killServer(pid);
waitForPort(port);
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
// ── Fetch ─────────────────────────────────────────────
auto cacheDir = makeTempDir("cache-happy");
scope (exit) removeDir(cacheDir);
auto cfg = testConfig(baseUrl, cacheDir);
auto recipePath = fetchRecipe("hello", cfg);
// Verify recipe path returned
assert(recipePath == cacheDir ~ "/recipes/hello/hello.recipe",
"Expected recipe path, got: " ~ recipePath);
// Verify all three files exist in cache
bool allOk = () @trusted {
return exists(cacheDir ~ "/recipes/hello/hello.recipe")
&& exists(cacheDir ~ "/recipes/hello/package.lua")
&& exists(cacheDir ~ "/recipes/hello/build.sh");
}();
assert(allOk, "All three files should exist in cache");
}
// ── Test 2: 404 on .recipe → FetchException "package not found"
@safe unittest
{
auto serveDir = makeTempDir("nf");
scope (exit) removeDir(serveDir);
// Empty recipes directory — no hello dir → 404 on .recipe
() @trusted { mkdirRecurse(buildPath(serveDir, "recipes")); }();
auto port = findFreePort();
auto pid = () @trusted {
return spawnProcess(
["python3", "-m", "http.server", port.to!string,
"--bind", "127.0.0.1"],
workDir: serveDir);
}();
scope (exit) killServer(pid);
waitForPort(port);
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
auto cacheDir = makeTempDir("cache-nf");
scope (exit) removeDir(cacheDir);
auto cfg = testConfig(baseUrl, cacheDir);
bool caught = false;
try
{
fetchRecipe("hello", cfg);
assert(false, "Expected FetchException");
}
catch (FetchException e)
{
caught = true;
assert(e.msg.indexOf("package 'hello' not found in ZUUR recipes") >= 0,
"Expected 'package not found' message, got: " ~ e.msg);
}
assert(caught, "Should have thrown FetchException");
}
// ── Test 3: custom build system — build_script fetched from subdir
@safe unittest
{
auto serveDir = makeTempDir("custom");
scope (exit) removeDir(serveDir);
writeTestFile(serveDir, "recipes/hello/hello.recipe",
`return {
name = "hello",
build_system = "custom",
build_script = "scripts/build.sh"
}`);
writeTestFile(serveDir, "recipes/hello/scripts/build.sh",
"#!/bin/sh\necho custom build");
auto port = findFreePort();
auto pid = () @trusted {
return spawnProcess(
["python3", "-m", "http.server", port.to!string,
"--bind", "127.0.0.1"],
workDir: serveDir);
}();
scope (exit) killServer(pid);
waitForPort(port);
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
auto cacheDir = makeTempDir("cache-custom");
scope (exit) removeDir(cacheDir);
auto cfg = testConfig(baseUrl, cacheDir);
auto recipePath = fetchRecipe("hello", cfg);
// Verify custom build script was downloaded
bool scriptOk = () @trusted {
return exists(cacheDir ~ "/recipes/hello/hello.recipe")
&& exists(cacheDir ~ "/recipes/hello/scripts/build.sh");
}();
assert(scriptOk,
"Both .recipe and scripts/build.sh should exist in cache");
}
// ── Test 4: 404 on package.lua / build.sh → no error, recipe still fetched
@safe unittest
{
auto serveDir = makeTempDir("opt404");
scope (exit) removeDir(serveDir);
// Only the .recipe file exists — no package.lua or build.sh
writeTestFile(serveDir, "recipes/hello/hello.recipe",
"return { name = 'hello', build_system = 'make' }");
auto port = findFreePort();
auto pid = () @trusted {
return spawnProcess(
["python3", "-m", "http.server", port.to!string,
"--bind", "127.0.0.1"],
workDir: serveDir);
}();
scope (exit) killServer(pid);
waitForPort(port);
auto baseUrl = "http://127.0.0.1:" ~ port.to!string;
auto cacheDir = makeTempDir("cache-opt404");
scope (exit) removeDir(cacheDir);
auto cfg = testConfig(baseUrl, cacheDir);
auto recipePath = fetchRecipe("hello", cfg);
// Should succeed — recipe downloaded, optional files merely warned
assert(recipePath == cacheDir ~ "/recipes/hello/hello.recipe");
bool recipeExists = () @trusted {
return exists(cacheDir ~ "/recipes/hello/hello.recipe");
}();
assert(recipeExists, "Recipe file should exist");
}
// ── Test 5: recipe download connection error → no partial files in cache
@safe unittest
{
auto serveDir = makeTempDir("fail");
scope (exit) removeDir(serveDir);
// Start a server that serves files, but point the config at a
// different (dead) port so the recipe download fails with a
// connection error.
auto port = findFreePort();
auto pid = () @trusted {
return spawnProcess(
["python3", "-m", "http.server", port.to!string,
"--bind", "127.0.0.1"],
workDir: serveDir);
}();
scope (exit) killServer(pid);
waitForPort(port);
// Use a dead port (nothing listening) for the recipe URL
auto deadPort = findFreePort(); // closed immediately, nothing listening
auto deadUrl = "http://127.0.0.1:" ~ deadPort.to!string;
auto cacheDir = makeTempDir("cache-fail");
scope (exit) removeDir(cacheDir);
auto cfg = testConfig(deadUrl, cacheDir);
bool caught = false;
try
{
fetchRecipe("hello", cfg);
assert(false, "Expected FetchException for connection error");
}
catch (FetchException e)
{
caught = true;
assert(e.msg.length > 0, "Should have an error message");
}
assert(caught, "Should have thrown FetchException");
// Verify no partial files were left in the cache directory
bool cacheClean = () @trusted {
// The cache dir may exist (mkdirRecurse) but should have no files
auto recipeDir = cacheDir ~ "/recipes/hello";
if (!exists(recipeDir))
return true;
// Check for files inside
import std.file : dirEntries, SpanMode;
foreach (de; dirEntries(recipeDir, SpanMode.shallow))
return false; // any entry = not clean
return true;
}();
assert(cacheClean,
"Cache directory should be clean after failed download");
}