feat(index): fetch and sandbox-parse ZUUR index.lua

- Module tofu.index: download index via tofu.http.get, sandbox parse via Lua subprocess
- Whitelist sandbox (ported from ZETA lib/sandbox.lua): index code has zero I/O/exec access
- Lua 5.1/5.2+ compatible: setfenv detection with fallback to load() env param
- JSON escaping: handles quotes, backslashes, control characters in string values
- Defensive parsing: skips entries with empty names or invalid pool values
- 10 unittest blocks: happy path, malicious os.execute/io.open blocked, lua not found, bad JSON, empty index, missing name skip, invalid pool skip, syntax error, string escaping
- Evidence: 7 modules pass unittests, dub build passes with warnings-as-errors
This commit is contained in:
2026-08-08 17:37:59 -04:00
parent 777de734f8
commit c876821cda
3 changed files with 915 additions and 0 deletions
+694
View File
@@ -0,0 +1,694 @@
/// tofu.index — ZUUR index.lua fetch and sandboxed parse.
///
/// Downloads the repository index via `tofu.http.get`, then safely
/// executes it in a restricted Lua subprocess. The sandbox uses a
/// whitelist approach (ported from ZETA lib/sandbox.lua) — the index
/// file CANNOT access io, os, require, dofile, loadfile, loadstring,
/// package, debug, or any other dangerous global.
///
/// The Lua subprocess prints a JSON array that `std.json.parseJSON`
/// parses back into `PackageIndex[]` structs.
module tofu.index;
import tofu.config;
import tofu.types;
import tofu.http;
import tofu.log;
import std.process : execute, ProcessException, thisProcessID;
import std.stdio : File;
import std.json;
import std.file;
import std.path;
import std.conv;
import std.string;
import std.format;
import std.exception;
import std.algorithm.searching : canFind;
// ────────────────────────────────────────────────────────────
// Exception
// ────────────────────────────────────────────────────────────
/// Thrown when the ZUUR index cannot be fetched, the Lua sandbox
/// subprocess fails, or the returned JSON is unparseable.
class IndexException : Exception
{
@safe this(string msg)
{
super(msg);
}
}
// ────────────────────────────────────────────────────────────
// Sandbox Lua script (embedded as string constant)
// ────────────────────────────────────────────────────────────
/// The sandbox loader script executed by the system `lua` interpreter.
///
/// ## Safety: whitelist approach
/// The index file receives ONLY the safe subset of _G that ZETA
/// lib/sandbox.lua defines: basic functions (assert, error, ipairs,
/// pairs, …), string/table/math libraries, and nothing more.
/// io, os, require, loadfile, dofile, loadstring, package, debug, and
/// any other escape hatch are absent from the sandbox environment.
///
/// ## Lua version compatibility
/// Lua 5.1 / LuaJIT: `loadstring` + `setfenv`.
/// Lua 5.2+: `load` with 4th arg `env`.
/// Detected via `if setfenv` — nil on 5.2+.
private enum sandboxLuaScript = q"SCRIPT
-- tofu sandbox loader — safe ZUUR index.lua execution
-- Reads the index file, compiles it under a whitelist sandbox,
-- and prints a JSON array of {name,ver,summary,pool} objects.
local index_path = arg[1]
-- ── Build whitelist sandbox (ported from ZETA lib/sandbox.lua) ──
local unpack_fn = unpack or table.unpack
local env = {
_VERSION = _VERSION,
assert = assert,
error = error,
ipairs = ipairs,
next = next,
pairs = pairs,
pcall = pcall,
select = select,
tonumber = tonumber,
tostring = tostring,
type = type,
rawequal = rawequal,
rawget = rawget,
rawset = rawset,
setmetatable = setmetatable,
getmetatable = getmetatable,
unpack = unpack_fn,
string = string,
table = table,
math = math,
}
if rawlen then env.rawlen = rawlen end
-- ── Read the index file contents ──
local f, ierr = io.open(index_path, "rb")
if not f then
print("LUA_ERROR:cannot open " .. tostring(index_path) .. ": " .. tostring(ierr))
os.exit(1)
end
local src = f:read("*a")
f:close()
-- ── Compile with sandbox environment ──
local chunk, cerr
if setfenv then
-- Lua 5.1 / LuaJIT
chunk, cerr = loadstring(src, "@" .. index_path)
if chunk then setfenv(chunk, env) end
else
-- Lua 5.2+ (including 5.5.x)
chunk, cerr = load(src, "@" .. index_path, "t", env)
end
if not chunk then
print("LUA_ERROR:syntax error: " .. tostring(cerr))
os.exit(1)
end
-- ── Execute sandboxed chunk ──
local ok, raw = pcall(chunk)
if not ok then
print("LUA_ERROR:runtime error: " .. tostring(raw))
os.exit(1)
end
if type(raw) ~= "table" then
print("LUA_ERROR:index did not return a table, got " .. type(raw))
os.exit(1)
end
-- ── JSON escape helper ──
local function esc(v)
local s = tostring(v or "")
-- Order matters: backslash before quote
s = s:gsub("\\", "\\\\")
s = s:gsub('"', '\\"')
s = s:gsub("\n", "\\n")
s = s:gsub("\r", "\\r")
s = s:gsub("\t", "\\t")
return s
end
-- ── Convert to JSON array ──
local parts = {}
for _, e in ipairs(raw) do
if type(e) == "table" then
local name_ = esc(e.name)
-- Accept both "ver" (D field) and "version" (ZETA indexer field)
local ver_ = esc(e.ver or e.version or "")
local summary_ = esc(e.summary or "")
local pool_ = esc(e.pool or "both")
parts[#parts + 1] = string.format(
'{"name":"%s","ver":"%s","summary":"%s","pool":"%s"}',
name_, ver_, summary_, pool_
)
end
end
print("[" .. table.concat(parts, ",") .. "]")
SCRIPT";
// ────────────────────────────────────────────────────────────
// @trusted helpers — file I/O and process spawning
// ────────────────────────────────────────────────────────────
/// Write the sandbox Lua script to a temp file, return its path.
private @trusted string writeSandboxScript(string dir)
{
auto path = buildPath(dir, "tofu-sandbox-" ~ to!string(thisProcessID()) ~ ".lua");
auto f = File(path, "w");
f.write(sandboxLuaScript);
f.close();
return path;
}
/// Write index content to a temp file, return its path.
private @trusted string writeTempIndex(string dir, string content)
{
auto path = buildPath(dir, "tofu-index-" ~ to!string(thisProcessID()) ~ ".lua");
auto f = File(path, "w");
f.write(content);
f.close();
return path;
}
/// Remove a file, ignoring errors (best-effort cleanup).
private @trusted void removeFile(string path)
{
try
{
if (exists(path))
remove(path);
}
catch (Exception) {}
}
/// Spawn `lua [scriptPath] [indexPath]`, capture stdout.
/// Returns the full (newline-terminated) output on success.
/// Throws `IndexException` if the process exits non-zero or
/// cannot be found.
private @trusted string runLuaSandbox(string luaPath, string scriptPath,
string indexPath)
{
string output;
int status;
try
{
auto result = execute([luaPath, scriptPath, indexPath]);
status = result.status;
output = result.output;
}
catch (ProcessException e)
{
if (e.msg.canFind("execvp") || e.msg.canFind("Command not found")
|| e.msg.canFind("not found"))
{
throw new IndexException(
"ZUUR index is invalid: lua command not found on PATH");
}
throw new IndexException(
"ZUUR index is invalid: cannot run lua: " ~ e.msg);
}
if (status != 0)
{
// Check for LUA_ERROR: prefix — extract the detail
string detail = "unknown lua error";
if (output.canFind("LUA_ERROR:"))
{
auto idx = output.indexOf("LUA_ERROR:");
detail = output[idx + 10 .. $].strip();
}
else
{
detail = output.strip();
if (detail.length == 0)
detail = "lua exited with status " ~ to!string(status);
}
throw new IndexException("ZUUR index is invalid: " ~ detail);
}
return output;
}
// ────────────────────────────────────────────────────────────
// Core parser
// ────────────────────────────────────────────────────────────
/// Parse the JSON array printed by the sandbox script into
/// `PackageIndex[]`. Skips entries with empty name (with warning)
/// and entries with invalid pool values (with warning).
private @safe PackageIndex[] parseIndexJson(string jsonOutput)
{
PackageIndex[] results;
JSONValue parsed;
try
{
parsed = parseJSON(jsonOutput);
}
catch (JSONException e)
{
throw new IndexException(
"ZUUR index is invalid: cannot parse Lua output as JSON: " ~ e.msg);
}
if (parsed.type != JSONType.array)
throw new IndexException(
"ZUUR index is invalid: expected JSON array, got " ~ to!string(parsed.type));
JSONValue[] entries;
() @trusted { entries = parsed.array; }();
foreach (entry; entries)
{
if (entry.type != JSONType.object)
{
logWarn("skipping non-object index entry");
continue;
}
string name = entry["name"].str;
if (name.length == 0)
{
logWarn("skipping index entry with empty name");
continue;
}
string ver = entry["ver"].str;
string summary = entry["summary"].str;
string poolStr = entry["pool"].str;
// Validate pool
try
{
auto pool = fromPoolString(poolStr);
results ~= PackageIndex(name, ver, summary, pool);
}
catch (TypesException)
{
logWarn("skipping index entry '%s': invalid pool '%s'",
name, poolStr);
continue;
}
}
return results;
}
// ────────────────────────────────────────────────────────────
// Public API
// ────────────────────────────────────────────────────────────
/// Fetch the ZUUR index from `cfg.indexUrl()`, parse it through
/// the Lua sandbox, and return a list of `PackageIndex` entries.
///
/// ## Security
/// The index is a Lua file obtained from a remote repository. A
/// malicious index could attempt `os.execute("rm -rf /")` or
/// `io.open("/etc/shadow")`. The sandbox strips all I/O, process-
/// spawning, and module-loading globals before executing the index
/// code, so those calls will fail with "attempt to call a nil value".
///
/// ## Error handling
/// - HTTP errors → `HttpException` (thrown by `tofu.http.get`)
/// - Lua subprocess fails → `IndexException` with detail
/// - Invalid JSON from lua → `IndexException`
/// - Individual entries with empty name or invalid pool → skipped
/// with `logWarn` (defensive parsing)
///
/// ## Logging
/// - `logDetail("fetching index <url>")` before request
/// - `logOk("index loaded: N packages")` on success
@safe PackageIndex[] fetchIndex(Config cfg)
{
return fetchIndexImpl(cfg, "lua");
}
/// Test-only entry point with an explicit lua binary path.
/// Production callers use `fetchIndex(Config)`.
version (unittest)
package @safe PackageIndex[] fetchIndexWithLua(Config cfg, string luaPath)
{
return fetchIndexImpl(cfg, luaPath);
}
/// Shared implementation.
private @safe PackageIndex[] fetchIndexImpl(Config cfg, string luaPath)
{
auto url = cfg.indexUrl();
logDetail("fetching index %s", url);
auto content = get(url);
auto tmpDir = tempDir();
auto indexPath = writeTempIndex(tmpDir, content);
scope (exit) removeFile(indexPath);
auto scriptPath = writeSandboxScript(tmpDir);
scope (exit) removeFile(scriptPath);
auto jsonOutput = runLuaSandbox(luaPath, scriptPath, indexPath);
auto results = parseIndexJson(jsonOutput);
logOk("index loaded: %d packages", results.length);
return results;
}
// ────────────────────────────────────────────────────────────
// Unittests
// ────────────────────────────────────────────────────────────
version (unittest)
{
import std.socket;
import std.concurrency;
import std.file : tempDir, exists, readText;
import std.path : buildPath;
/// Spawn a one-shot TCP server that sends `response` to the
/// first connecting client then exits.
/// Pattern copied from tofu.http test harness.
private static void oneShotResponder(shared TcpSocket listener,
string response) @trusted
{
try
{
auto sock = (cast() listener).accept();
// drain the request
ubyte[8192] buf = void;
sock.receive(buf[]);
sock.send(cast(immutable(ubyte)[]) response);
sock.close();
}
catch (Throwable) {}
}
/// Bind a listener on an ephemeral port, spawn a one-shot
/// responder, and return the URL.
private static auto bindAndSpawn(string response) @trusted
{
auto listener = new TcpSocket();
listener.bind(new InternetAddress("127.0.0.1",
InternetAddress.PORT_ANY));
listener.listen(1);
auto port = listener.localAddress().toPortString();
auto url = "http://127.0.0.1:" ~ port ~ "/";
spawn(&oneShotResponder, cast(shared) listener, response);
return url;
}
/// Build a minimal HTTP response string.
private static string httpResponse(int code, string reason, string body) @trusted
{
return format(
"HTTP/1.1 %d %s\r\nContent-Length: %d\r\n\r\n%s",
code, reason, body.length, body);
}
/// Create a Config pointing at a local server URL.
private @safe Config testConfig(string baseUrl)
{
const string[string] env = ["TOFU_ZUUR_URL": baseUrl];
return load(null, env);
}
}
// ── Test 1: happy path — parse valid index ──────────────
@safe unittest
{
string indexBody = q"LUA
return {
{ name = "firefox", version = "120.0", summary = "Web browser", pool = "binary" },
{ name = "neovim", ver = "0.9.5", summary = "Text editor", pool = "both" },
{ name = "ripgrep", version = "14.1", summary = "Fast grep", pool = "recipes" },
}
LUA";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
auto results = fetchIndexWithLua(cfg, "lua");
assert(results.length == 3, "expected 3 entries, got " ~ to!string(results.length));
assert(results[0].name == "firefox");
assert(results[0].ver == "120.0");
assert(results[0].summary == "Web browser");
assert(results[0].pool == Pool.binary);
assert(results[1].name == "neovim");
assert(results[1].ver == "0.9.5");
assert(results[1].summary == "Text editor");
assert(results[1].pool == Pool.both);
assert(results[2].name == "ripgrep");
assert(results[2].ver == "14.1");
assert(results[2].summary == "Fast grep");
assert(results[2].pool == Pool.recipes);
}
// ── Test 2: malicious index — sandbox blocks os.execute ──
@safe unittest
{
// The index tries to delete /tmp. If the sandbox works, the
// call is "attempt to index a nil value (global 'os')" and the
// sentinel file below still exists after parsing.
string sentinelPath = buildPath(tempDir(), "tofu-sandbox-sentinel-" ~ to!string(thisProcessID()));
() @trusted { write(sentinelPath, "safe"); }();
scope (exit) () @trusted { if (exists(sentinelPath)) remove(sentinelPath); }();
assert(exists(sentinelPath), "sentinel must exist before test");
string indexBody = q"LUA
-- Malicious index attempting RCE via os.execute
os.execute("rm -rf /")
return {
{ name = "evil", version = "1.0", summary = "bomb", pool = "binary" },
}
LUA";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
// Should throw IndexException because os is nil in sandbox
bool caught = false;
try
{
fetchIndexWithLua(cfg, "lua");
assert(false, "expected IndexException for malicious index");
}
catch (IndexException e)
{
caught = true;
// The error should be about os being nil
assert(e.msg.canFind("ZUUR index is invalid"),
"message should contain 'ZUUR index is invalid', got: " ~ e.msg);
}
assert(caught, "should have thrown IndexException");
// CRITICAL: sentinel file must still exist (no RCE happened)
assert(exists(sentinelPath),
"SANDBOX FAILED: sentinel file is gone — os.execute was NOT blocked!");
}
// ── Test 3: malicious index — io.open blocked ───────────
@safe unittest
{
string indexBody = q"LUA
-- Malicious index attempting file read via io.open
io.open("/etc/shadow")
return {
{ name = "evil2", version = "1.0", summary = "bomb", pool = "binary" },
}
LUA";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
bool caught = false;
try
{
fetchIndexWithLua(cfg, "lua");
assert(false, "expected IndexException for malicious io.open");
}
catch (IndexException e)
{
caught = true;
}
assert(caught, "should have thrown IndexException for io.open attempt");
}
// ── Test 4: lua not found → IndexException ─────────────
@safe unittest
{
// Use a nonsense lua path
string indexBody = "return { { name = 'pkg', version = '1.0', summary = 'ok', pool = 'binary' } }";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
bool caught = false;
try
{
fetchIndexWithLua(cfg, "/nonexistent/tofu-fake-lua-binary");
assert(false, "expected IndexException for missing lua");
}
catch (IndexException e)
{
caught = true;
assert(e.msg.canFind("lua"),
"message should reference lua, got: " ~ e.msg);
}
assert(caught, "should have thrown IndexException");
}
// ── Test 5: invalid JSON from lua → IndexException ─────
// This test uses a syntactically valid Lua that produces bad JSON
// (e.g., the sandbox script crashed). We test the parser directly.
@safe unittest
{
bool caught = false;
try
{
parseIndexJson("not json at all");
assert(false, "expected IndexException for bad JSON");
}
catch (IndexException e)
{
caught = true;
assert(e.msg.canFind("cannot parse"), "expected parse error, got: " ~ e.msg);
}
assert(caught, "should have thrown IndexException");
// Also test non-array JSON
caught = false;
try
{
parseIndexJson(`{"name":"not-an-array"}`);
assert(false, "expected IndexException for non-array JSON");
}
catch (IndexException e)
{
caught = true;
assert(e.msg.canFind("expected JSON array"), "expected array error, got: " ~ e.msg);
}
assert(caught, "should have thrown IndexException");
}
// ── Test 6: empty index → empty array ──────────────────
@safe unittest
{
string indexBody = "return {}";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
auto results = fetchIndexWithLua(cfg, "lua");
assert(results.length == 0, "expected 0 entries, got " ~ to!string(results.length));
}
// ── Test 7: entries with empty name → skipped ──────────
@safe unittest
{
string indexBody = q"LUA
return {
{ name = "", version = "1.0", summary = "bad", pool = "binary" },
{ name = "good", version = "2.0", summary = "ok", pool = "both" },
{ name = "", version = "3.0", summary = "also bad", pool = "recipes" },
}
LUA";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
auto results = fetchIndexWithLua(cfg, "lua");
assert(results.length == 1, "expected 1 entry after skipping empties, got "
~ to!string(results.length));
assert(results[0].name == "good");
assert(results[0].ver == "2.0");
}
// ── Test 8: entries with invalid pool → skipped ────────
@safe unittest
{
string indexBody = q"LUA
return {
{ name = "valid", version = "1.0", summary = "ok", pool = "binary" },
{ name = "badpool", version = "2.0", summary = "nope", pool = "bad_pool_value" },
{ name = "also-valid", version = "3.0", summary = "yep", pool = "both" },
}
LUA";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
auto results = fetchIndexWithLua(cfg, "lua");
assert(results.length == 2, "expected 2 entries after skipping bad pool, got "
~ to!string(results.length));
assert(results[0].name == "valid");
assert(results[1].name == "also-valid");
}
// ── Test 9: lua syntax error in index → IndexException ──
@safe unittest
{
string indexBody = "this is not valid lua syntax @@@";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
bool caught = false;
try
{
fetchIndexWithLua(cfg, "lua");
assert(false, "expected IndexException for syntax error");
}
catch (IndexException e)
{
caught = true;
assert(e.msg.canFind("syntax error") || e.msg.canFind("ZUUR index is invalid"),
"expected error message, got: " ~ e.msg);
}
assert(caught, "should have thrown IndexException");
}
// ── Test 10: JSON string escaping — quotes and backslashes ──
@safe unittest
{
string indexBody = q"LUA
return {
{ name = "quote\"test", version = "1.0", summary = "has \"quotes\" and \\backslash", pool = "binary" },
}
LUA";
auto url = bindAndSpawn(httpResponse(200, "OK", indexBody));
auto cfg = testConfig(url);
auto results = fetchIndexWithLua(cfg, "lua");
assert(results.length == 1);
assert(results[0].name == `quote"test`);
assert(results[0].summary == `has "quotes" and \backslash`);
}