feat(http): add synchronous HTTP client with std.net.curl
This commit is contained in:
+356
@@ -0,0 +1,356 @@
|
||||
/// tofu.http — synchronous HTTP client using `std.net.curl`.
|
||||
///
|
||||
/// Provides `get()` and `downloadFile()` with configurable timeouts,
|
||||
/// redirect following, and a custom User-Agent. All public APIs are
|
||||
/// `@safe`; the `@system` curl internals are isolated behind `@trusted`
|
||||
/// helpers.
|
||||
///
|
||||
/// URL scheme: Both `http://` and `https://` URLs are accepted. The
|
||||
/// test harness requires `http://` for local mock servers; production
|
||||
/// configuration points at HTTPS.
|
||||
///
|
||||
/// Callers catch `HttpException` and translate to appropriate exit
|
||||
/// codes — this module does NOT log; it only throws.
|
||||
module tofu.http;
|
||||
|
||||
import std.net.curl;
|
||||
import std.file;
|
||||
import std.path;
|
||||
import std.conv;
|
||||
import std.stdio;
|
||||
import core.time;
|
||||
import std.format;
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Exception
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Thrown on any HTTP error (status >= 400), connection failure,
|
||||
/// timeout, or curl-level error.
|
||||
class HttpException : Exception
|
||||
{
|
||||
@safe this(string msg)
|
||||
{
|
||||
super(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Public API (@safe — @trusted wrappers isolate curl)
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Perform a GET request to `url` and return the response body as a string.
|
||||
///
|
||||
/// Throws `HttpException` on:
|
||||
/// - HTTP status code >= 400 (includes status code and url)
|
||||
/// - Connection failure
|
||||
/// - Timeout (connect 30s, operation 120s)
|
||||
/// - Any curl-level error
|
||||
@safe string get(string url)
|
||||
{
|
||||
return getImpl(url);
|
||||
}
|
||||
|
||||
/// Download `url` to `destPath` on disk.
|
||||
///
|
||||
/// Creates parent directories as needed. Downloads to a `.part` temp
|
||||
/// file first, renames on success, and removes the partial file on
|
||||
/// any error. Returns `destPath` on success.
|
||||
///
|
||||
/// Throws `HttpException` on any HTTP, connection, or timeout error.
|
||||
@safe string downloadFile(string url, string destPath)
|
||||
{
|
||||
return downloadFileImpl(url, destPath);
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Constants
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
private enum uint maxRedirectsCount = 5;
|
||||
private enum string userAgentString = "tofu/0.1.0";
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// @trusted helpers — all @system curl calls are isolated here
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Apply common settings to an HTTP handle.
|
||||
private @trusted void configure(HTTP http)
|
||||
{
|
||||
http.connectTimeout = 30.seconds;
|
||||
http.operationTimeout = 120.seconds;
|
||||
http.maxRedirects = maxRedirectsCount;
|
||||
http.setUserAgent(userAgentString);
|
||||
}
|
||||
|
||||
/// Core GET implementation — all curl interaction happens here.
|
||||
private @trusted string getImpl(string url)
|
||||
{
|
||||
auto http = HTTP();
|
||||
configure(http);
|
||||
http.url = url;
|
||||
|
||||
string content;
|
||||
int statusCode;
|
||||
|
||||
http.onReceive = (ubyte[] data)
|
||||
{
|
||||
content ~= cast(string) data;
|
||||
return data.length;
|
||||
};
|
||||
|
||||
http.onReceiveStatusLine = (HTTP.StatusLine l)
|
||||
{
|
||||
statusCode = l.code;
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
http.perform();
|
||||
}
|
||||
catch (CurlTimeoutException e)
|
||||
{
|
||||
throw new HttpException(
|
||||
"cannot reach ZUUR at " ~ url ~ ": timeout after 120s");
|
||||
}
|
||||
catch (CurlException e)
|
||||
{
|
||||
throw new HttpException(e.msg);
|
||||
}
|
||||
|
||||
if (statusCode >= 400)
|
||||
{
|
||||
throw new HttpException(
|
||||
format("HTTP %d fetching %s", statusCode, url));
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
/// Core download implementation — .part temp file + rename.
|
||||
private @trusted string downloadFileImpl(string url, string destPath)
|
||||
{
|
||||
import std.file : mkdirRecurse, rename, exists, remove;
|
||||
|
||||
// Create parent directories
|
||||
auto dir = destPath.dirName;
|
||||
if (dir.length > 0 && !exists(dir))
|
||||
mkdirRecurse(dir);
|
||||
|
||||
auto tmpPath = destPath ~ ".part";
|
||||
|
||||
auto f = File(tmpPath, "wb");
|
||||
scope (failure)
|
||||
{
|
||||
if (exists(tmpPath))
|
||||
remove(tmpPath);
|
||||
}
|
||||
|
||||
auto http = HTTP();
|
||||
configure(http);
|
||||
http.url = url;
|
||||
|
||||
int statusCode;
|
||||
|
||||
http.onReceive = (ubyte[] data)
|
||||
{
|
||||
f.rawWrite(data);
|
||||
return data.length;
|
||||
};
|
||||
|
||||
http.onReceiveStatusLine = (HTTP.StatusLine l)
|
||||
{
|
||||
statusCode = l.code;
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
http.perform();
|
||||
f.close();
|
||||
}
|
||||
catch (CurlTimeoutException e)
|
||||
{
|
||||
throw new HttpException(
|
||||
"cannot reach ZUUR at " ~ url ~ ": timeout after 120s");
|
||||
}
|
||||
catch (CurlException e)
|
||||
{
|
||||
throw new HttpException(e.msg);
|
||||
}
|
||||
|
||||
if (statusCode >= 400)
|
||||
{
|
||||
throw new HttpException(
|
||||
format("HTTP %d fetching %s", statusCode, url));
|
||||
}
|
||||
|
||||
// Atomically rename partial → final
|
||||
try
|
||||
{
|
||||
if (exists(destPath))
|
||||
remove(destPath);
|
||||
rename(tmpPath, destPath);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
if (exists(tmpPath))
|
||||
remove(tmpPath);
|
||||
throw new HttpException(
|
||||
"Failed to finalize download to " ~ destPath ~ ": " ~ e.msg);
|
||||
}
|
||||
|
||||
return destPath;
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Unittests
|
||||
// ────────────────────────────────────────────────────────────
|
||||
|
||||
version (unittest)
|
||||
{
|
||||
import std.socket;
|
||||
import std.concurrency;
|
||||
import std.string;
|
||||
import std.algorithm.searching : canFind;
|
||||
|
||||
/// Spawn a one-shot TCP server that sends `response` to the
|
||||
/// first connecting client then exits.
|
||||
private static void oneShotResponder(shared TcpSocket listener,
|
||||
string response) @trusted
|
||||
{
|
||||
try
|
||||
{
|
||||
auto sock = (cast() listener).accept();
|
||||
// drain the request
|
||||
ubyte[8192] buf = void;
|
||||
sock.receive(buf[]);
|
||||
sock.send(cast(immutable(ubyte)[]) response);
|
||||
sock.close();
|
||||
}
|
||||
catch (Throwable) {}
|
||||
}
|
||||
|
||||
/// Bind a listener on an ephemeral port and return the URL.
|
||||
private static auto bindAndSpawn(string response) @trusted
|
||||
{
|
||||
auto listener = new TcpSocket();
|
||||
listener.bind(new InternetAddress("127.0.0.1",
|
||||
InternetAddress.PORT_ANY));
|
||||
listener.listen(1);
|
||||
auto port = listener.localAddress().toPortString();
|
||||
auto url = "http://127.0.0.1:" ~ port ~ "/";
|
||||
spawn(&oneShotResponder, cast(shared) listener, response);
|
||||
return url;
|
||||
}
|
||||
|
||||
/// Build a minimal HTTP response string.
|
||||
private static string httpResponse(int code, string reason, string body) @trusted
|
||||
{
|
||||
return format(
|
||||
"HTTP/1.1 %d %s\r\nContent-Length: %d\r\n\r\n%s",
|
||||
code, reason, body.length, body);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Test 1: GET against a local server, verify body ────
|
||||
@safe unittest
|
||||
{
|
||||
auto url = bindAndSpawn(httpResponse(200, "OK", "hello tofu"));
|
||||
auto body = get(url);
|
||||
assert(body == "hello tofu",
|
||||
"Expected 'hello tofu' but got: '" ~ body ~ "'");
|
||||
}
|
||||
|
||||
// ── Test 2: 404 → HttpException with correct message ────
|
||||
@safe unittest
|
||||
{
|
||||
auto url = bindAndSpawn(httpResponse(404, "Not Found", "gone"));
|
||||
bool caught = false;
|
||||
try
|
||||
{
|
||||
get(url);
|
||||
assert(false, "Expected HttpException for 404");
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
caught = true;
|
||||
assert(e.msg.canFind("HTTP 404"),
|
||||
"Message should contain 'HTTP 404', got: " ~ e.msg);
|
||||
assert(e.msg.canFind(url),
|
||||
"Message should contain url, got: " ~ e.msg);
|
||||
}
|
||||
assert(caught, "Should have thrown HttpException");
|
||||
}
|
||||
|
||||
// ── Test 3: downloadFile writes complete file ───────────
|
||||
@safe unittest
|
||||
{
|
||||
import std.file : tempDir, exists, readText;
|
||||
import std.path : buildPath;
|
||||
import std.process : thisProcessID;
|
||||
|
||||
auto url = bindAndSpawn(httpResponse(200, "OK", "downloaded content!"));
|
||||
auto dest = buildPath(tempDir(), "tofu_test_dl_" ~ to!string(thisProcessID()));
|
||||
scope (exit)
|
||||
{
|
||||
if (exists(dest))
|
||||
(() @trusted => remove(dest))();
|
||||
}
|
||||
|
||||
auto result = downloadFile(url, dest);
|
||||
assert(result == dest, "Should return destPath");
|
||||
assert(exists(dest), "File should exist after download");
|
||||
|
||||
string content;
|
||||
(() @trusted { content = readText(dest); })();
|
||||
assert(content == "downloaded content!",
|
||||
"File content mismatch, got: '" ~ content ~ "'");
|
||||
}
|
||||
|
||||
// ── Test 4: connection refused → HttpException ──────────
|
||||
@safe unittest
|
||||
{
|
||||
import std.socket;
|
||||
|
||||
// Get an ephemeral port then close it so nothing listens
|
||||
auto dead = new TcpSocket();
|
||||
dead.bind(new InternetAddress("127.0.0.1", InternetAddress.PORT_ANY));
|
||||
auto port = dead.localAddress().toPortString();
|
||||
dead.close();
|
||||
|
||||
auto url = "http://127.0.0.1:" ~ port ~ "/";
|
||||
bool caught = false;
|
||||
try
|
||||
{
|
||||
get(url);
|
||||
assert(false, "Expected HttpException for refused connection");
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
caught = true;
|
||||
// Message should contain some curl-level error description
|
||||
assert(e.msg.length > 0);
|
||||
}
|
||||
assert(caught, "Should have thrown HttpException");
|
||||
}
|
||||
|
||||
// ── Test 5: GET with 500 status → HttpException ─────────
|
||||
@safe unittest
|
||||
{
|
||||
auto url = bindAndSpawn(httpResponse(500, "Internal Server Error", "boom"));
|
||||
bool caught = false;
|
||||
try
|
||||
{
|
||||
get(url);
|
||||
assert(false, "Expected HttpException for 500");
|
||||
}
|
||||
catch (HttpException e)
|
||||
{
|
||||
caught = true;
|
||||
assert(e.msg.canFind("HTTP 500"),
|
||||
"Message should contain 'HTTP 500', got: " ~ e.msg);
|
||||
assert(e.msg.canFind(url),
|
||||
"Message should contain url, got: " ~ e.msg);
|
||||
}
|
||||
assert(caught, "Should have thrown HttpException");
|
||||
}
|
||||
Reference in New Issue
Block a user