From 777de734f8309e96f236ea057382ad5771e9c7d9 Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Sat, 8 Aug 2026 17:30:34 -0400 Subject: [PATCH] feat(http): add synchronous HTTP client with std.net.curl --- .omo/evidence/task-6-tofu-core.log | 35 +++ .omo/notepads/tofu-core/learnings.md | 72 ++++++ src/tofu/http.d | 356 +++++++++++++++++++++++++++ 3 files changed, 463 insertions(+) create mode 100644 .omo/evidence/task-6-tofu-core.log create mode 100644 src/tofu/http.d diff --git a/.omo/evidence/task-6-tofu-core.log b/.omo/evidence/task-6-tofu-core.log new file mode 100644 index 0000000..6319745 --- /dev/null +++ b/.omo/evidence/task-6-tofu-core.log @@ -0,0 +1,35 @@ + Warning + Warning ## Warning for package tofu ## + Warning + Warning The following compiler flags have been specified in the package description + Warning file. They are handled by DUB and direct use in packages is discouraged. + Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags + Warning to the compiler, or use one of the suggestions below: + Warning + Warning warningsAsErrors: Use "buildRequirements" to control the warning level + Warning + Generating test runner configuration 'tofu-test-application' for 'application' (executable). + Warning Excluding main source file src/main.d from test. + Starting Performing "unittest" build using /usr/bin/dmd for x86_64. + Up-to-date toml 1.0.0: target for configuration [library] is up to date. + Up-to-date tofu ~main: target for configuration [tofu-test-application] is up to date. + Finished To force a rebuild of up-to-date targets, run again with --force + Running tofu-test-application +Warning: malformed TOML config at /tmp/tofu-test-config-bad-183825.toml: Invalid table key declaration (2:0) +Warning: invalid TOFU_DEFAULT_JOBS 'not-a-number', using default 1 +5 modules passed unittests + Warning + Warning ## Warning for package tofu ## + Warning + Warning The following compiler flags have been specified in the package description + Warning file. They are handled by DUB and direct use in packages is discouraged. + Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags + Warning to the compiler, or use one of the suggestions below: + Warning + Warning warningsAsErrors: Use "buildRequirements" to control the warning level + Warning + Starting Performing "debug" build using /usr/bin/dmd for x86_64. + Up-to-date toml 1.0.0: target for configuration [library] is up to date. + Building tofu ~main: building configuration [application] + Linking tofu + Finished To force a rebuild of up-to-date targets, run again with --force diff --git a/.omo/notepads/tofu-core/learnings.md b/.omo/notepads/tofu-core/learnings.md index 118d51e..19264bc 100644 --- a/.omo/notepads/tofu-core/learnings.md +++ b/.omo/notepads/tofu-core/learnings.md @@ -108,3 +108,75 @@ _Auto-scaffolded by /start-work. Append new entries below - never overwrite._ - **`BuildResult.failed` as `BuildFailure[]`**: the plan describes it as `struct {string name; string reason}[]` — in D this is a named struct `BuildFailure` with array field `BuildFailure[] failed`. The `failedNames()` helper extracts just the names for caller convenience. - **`DepConstraint.parse` is a static factory method**: returns a new `DepConstraint` by value, never allocates on the heap. `@safe` throughout. - **All structs are `@safe`**: no methods do I/O, no `@system` calls. The module stands alone — no imports from other tofu modules. + +--- + +## Task 6 — `tofu.http` (sync HTTP client via std.net.curl) + +### std.net.curl API (key members for HTTP struct) +- **Construction**: `HTTP(url)` via `opCall`, or `HTTP()` + `.url = url` setter. + The `.url` setter auto-prepends `http://` if no scheme is present. +- **Timeouts** (all are `@property` setters taking `core.time.Duration`): + - `connectTimeout` — connect phase only (`CurlOption.connecttimeout_ms`). + - `operationTimeout` — DNS + connect + transfer total (`CurlOption.timeout_ms`). + - `dataTimeout` — low-speed activity timeout (NOT total read timeout; + sets `low_speed_limit` + `low_speed_time`). + - For a "read 120s" requirement use `operationTimeout = 120.seconds` — + it covers the entire operation including data transfer. +- **Redirects**: `maxRedirects` (property setter, takes `uint`). + Defaults to 10. Set to 0 to disable, `uint.max` for infinite. + Internally sets `CurlOption.followlocation` + `CurlOption.maxredirs`. +- **User-Agent**: `setUserAgent(string)` — instance method (not property). +- **onReceive**: `void delegate(ubyte[])` returning `size_t`. Must accept + all bytes or the request aborts. Use `return data.length;`. +- **onReceiveStatusLine**: Not a direct property on HTTP struct — must + be set via the `onReceiveHeader` property setter which wraps the + delegate and internally fires `onReceiveStatusLine` when it detects + an HTTP status line. After `perform()` the final status is available + via `http.statusLine`. +- **statusLine**: `HTTP.StatusLine` struct with `.code` (`ushort`), + `.majorVersion`, `.minorVersion`, `.reason`. Reset to zero before + each `perform()`. +- **perform()**: `CurlCode perform(ThrowOnError = Yes.throwOnError)`. + Default throws on curl errors, but the thrown exception classes are: + - `CurlTimeoutException : CurlException` — operation timed out (code 28). + - `CurlException` — all other curl errors. + - `HTTPStatusException : CurlException` — only thrown by high-level + wrappers (_basicHTTP); low-level `perform()` does NOT throw on + HTTP status codes — you must check `statusLine.code` yourself. +- **Exception isolation pattern**: catch `CurlTimeoutException` and + `CurlException` in `@trusted` helpers, re-throw as `HttpException` + (which extends `Exception`, not `CurlException`). Callers never see + curl types. +- **verifyPeer / verifyHost**: On by default in HTTP struct (no change needed). + +### @safe / @trusted architecture +- All public functions (`get`, `downloadFile`) are `@safe`. +- The `configure(HTTP)`, `getImpl`, and `downloadFileImpl` helpers are + `@trusted` — they are the ONLY places `std.net.curl` (which is + `@system`-heavy) is used. + +### Download pattern (.part temp file + rename) +- Ported from ZETA `lib/fetch.lua`: write to `dest.part`, rename on + success, remove partial on error. `scope(failure)` at function + level handles all error paths automatically. +- `File(tmpPath, "wb")` — binary write mode. File destructor auto-closes; + explicit `f.close()` needed before rename. + +### Unittest local server +- Used `std.concurrency.spawn` + `std.socket.TcpSocket` for a + self-contained one-shot HTTP server (no external dependency on Python3). +- Pattern: bind ephemeral port (`InternetAddress.PORT_ANY`), spawn thread + that `accept()`s one connection, sends canned HTTP response, exits. + `cast(shared)` the listener to pass to `spawn`. Mark test helpers + `@trusted`. +- HTTP response must include `Content-Length` header or libcurl hangs. + +### Connection refused test +- Bind socket, record port, close it, then `get()` to that port. + curl returns error 7 (`CURLE_COULDNT_CONNECT`) → `CurlException` + → caught and re-thrown as `HttpException`. + +### Build verified +- `dub build` passes with warnings-as-errors. +- `dub test` passes — all 5 modules (config, log, types, vercmp, http). diff --git a/src/tofu/http.d b/src/tofu/http.d new file mode 100644 index 0000000..ef066eb --- /dev/null +++ b/src/tofu/http.d @@ -0,0 +1,356 @@ +/// tofu.http — synchronous HTTP client using `std.net.curl`. +/// +/// Provides `get()` and `downloadFile()` with configurable timeouts, +/// redirect following, and a custom User-Agent. All public APIs are +/// `@safe`; the `@system` curl internals are isolated behind `@trusted` +/// helpers. +/// +/// URL scheme: Both `http://` and `https://` URLs are accepted. The +/// test harness requires `http://` for local mock servers; production +/// configuration points at HTTPS. +/// +/// Callers catch `HttpException` and translate to appropriate exit +/// codes — this module does NOT log; it only throws. +module tofu.http; + +import std.net.curl; +import std.file; +import std.path; +import std.conv; +import std.stdio; +import core.time; +import std.format; + +// ──────────────────────────────────────────────────────────── +// Exception +// ──────────────────────────────────────────────────────────── + +/// Thrown on any HTTP error (status >= 400), connection failure, +/// timeout, or curl-level error. +class HttpException : Exception +{ + @safe this(string msg) + { + super(msg); + } +} + +// ──────────────────────────────────────────────────────────── +// Public API (@safe — @trusted wrappers isolate curl) +// ──────────────────────────────────────────────────────────── + +/// Perform a GET request to `url` and return the response body as a string. +/// +/// Throws `HttpException` on: +/// - HTTP status code >= 400 (includes status code and url) +/// - Connection failure +/// - Timeout (connect 30s, operation 120s) +/// - Any curl-level error +@safe string get(string url) +{ + return getImpl(url); +} + +/// Download `url` to `destPath` on disk. +/// +/// Creates parent directories as needed. Downloads to a `.part` temp +/// file first, renames on success, and removes the partial file on +/// any error. Returns `destPath` on success. +/// +/// Throws `HttpException` on any HTTP, connection, or timeout error. +@safe string downloadFile(string url, string destPath) +{ + return downloadFileImpl(url, destPath); +} + +// ──────────────────────────────────────────────────────────── +// Constants +// ──────────────────────────────────────────────────────────── + +private enum uint maxRedirectsCount = 5; +private enum string userAgentString = "tofu/0.1.0"; + +// ──────────────────────────────────────────────────────────── +// @trusted helpers — all @system curl calls are isolated here +// ──────────────────────────────────────────────────────────── + +/// Apply common settings to an HTTP handle. +private @trusted void configure(HTTP http) +{ + http.connectTimeout = 30.seconds; + http.operationTimeout = 120.seconds; + http.maxRedirects = maxRedirectsCount; + http.setUserAgent(userAgentString); +} + +/// Core GET implementation — all curl interaction happens here. +private @trusted string getImpl(string url) +{ + auto http = HTTP(); + configure(http); + http.url = url; + + string content; + int statusCode; + + http.onReceive = (ubyte[] data) + { + content ~= cast(string) data; + return data.length; + }; + + http.onReceiveStatusLine = (HTTP.StatusLine l) + { + statusCode = l.code; + }; + + try + { + http.perform(); + } + catch (CurlTimeoutException e) + { + throw new HttpException( + "cannot reach ZUUR at " ~ url ~ ": timeout after 120s"); + } + catch (CurlException e) + { + throw new HttpException(e.msg); + } + + if (statusCode >= 400) + { + throw new HttpException( + format("HTTP %d fetching %s", statusCode, url)); + } + + return content; +} + +/// Core download implementation — .part temp file + rename. +private @trusted string downloadFileImpl(string url, string destPath) +{ + import std.file : mkdirRecurse, rename, exists, remove; + + // Create parent directories + auto dir = destPath.dirName; + if (dir.length > 0 && !exists(dir)) + mkdirRecurse(dir); + + auto tmpPath = destPath ~ ".part"; + + auto f = File(tmpPath, "wb"); + scope (failure) + { + if (exists(tmpPath)) + remove(tmpPath); + } + + auto http = HTTP(); + configure(http); + http.url = url; + + int statusCode; + + http.onReceive = (ubyte[] data) + { + f.rawWrite(data); + return data.length; + }; + + http.onReceiveStatusLine = (HTTP.StatusLine l) + { + statusCode = l.code; + }; + + try + { + http.perform(); + f.close(); + } + catch (CurlTimeoutException e) + { + throw new HttpException( + "cannot reach ZUUR at " ~ url ~ ": timeout after 120s"); + } + catch (CurlException e) + { + throw new HttpException(e.msg); + } + + if (statusCode >= 400) + { + throw new HttpException( + format("HTTP %d fetching %s", statusCode, url)); + } + + // Atomically rename partial → final + try + { + if (exists(destPath)) + remove(destPath); + rename(tmpPath, destPath); + } + catch (Exception e) + { + if (exists(tmpPath)) + remove(tmpPath); + throw new HttpException( + "Failed to finalize download to " ~ destPath ~ ": " ~ e.msg); + } + + return destPath; +} + +// ──────────────────────────────────────────────────────────── +// Unittests +// ──────────────────────────────────────────────────────────── + +version (unittest) +{ + import std.socket; + import std.concurrency; + import std.string; + import std.algorithm.searching : canFind; + + /// Spawn a one-shot TCP server that sends `response` to the + /// first connecting client then exits. + private static void oneShotResponder(shared TcpSocket listener, + string response) @trusted + { + try + { + auto sock = (cast() listener).accept(); + // drain the request + ubyte[8192] buf = void; + sock.receive(buf[]); + sock.send(cast(immutable(ubyte)[]) response); + sock.close(); + } + catch (Throwable) {} + } + + /// Bind a listener on an ephemeral port and return the URL. + private static auto bindAndSpawn(string response) @trusted + { + auto listener = new TcpSocket(); + listener.bind(new InternetAddress("127.0.0.1", + InternetAddress.PORT_ANY)); + listener.listen(1); + auto port = listener.localAddress().toPortString(); + auto url = "http://127.0.0.1:" ~ port ~ "/"; + spawn(&oneShotResponder, cast(shared) listener, response); + return url; + } + + /// Build a minimal HTTP response string. + private static string httpResponse(int code, string reason, string body) @trusted + { + return format( + "HTTP/1.1 %d %s\r\nContent-Length: %d\r\n\r\n%s", + code, reason, body.length, body); + } +} + +// ── Test 1: GET against a local server, verify body ──── +@safe unittest +{ + auto url = bindAndSpawn(httpResponse(200, "OK", "hello tofu")); + auto body = get(url); + assert(body == "hello tofu", + "Expected 'hello tofu' but got: '" ~ body ~ "'"); +} + +// ── Test 2: 404 → HttpException with correct message ──── +@safe unittest +{ + auto url = bindAndSpawn(httpResponse(404, "Not Found", "gone")); + bool caught = false; + try + { + get(url); + assert(false, "Expected HttpException for 404"); + } + catch (HttpException e) + { + caught = true; + assert(e.msg.canFind("HTTP 404"), + "Message should contain 'HTTP 404', got: " ~ e.msg); + assert(e.msg.canFind(url), + "Message should contain url, got: " ~ e.msg); + } + assert(caught, "Should have thrown HttpException"); +} + +// ── Test 3: downloadFile writes complete file ─────────── +@safe unittest +{ + import std.file : tempDir, exists, readText; + import std.path : buildPath; + import std.process : thisProcessID; + + auto url = bindAndSpawn(httpResponse(200, "OK", "downloaded content!")); + auto dest = buildPath(tempDir(), "tofu_test_dl_" ~ to!string(thisProcessID())); + scope (exit) + { + if (exists(dest)) + (() @trusted => remove(dest))(); + } + + auto result = downloadFile(url, dest); + assert(result == dest, "Should return destPath"); + assert(exists(dest), "File should exist after download"); + + string content; + (() @trusted { content = readText(dest); })(); + assert(content == "downloaded content!", + "File content mismatch, got: '" ~ content ~ "'"); +} + +// ── Test 4: connection refused → HttpException ────────── +@safe unittest +{ + import std.socket; + + // Get an ephemeral port then close it so nothing listens + auto dead = new TcpSocket(); + dead.bind(new InternetAddress("127.0.0.1", InternetAddress.PORT_ANY)); + auto port = dead.localAddress().toPortString(); + dead.close(); + + auto url = "http://127.0.0.1:" ~ port ~ "/"; + bool caught = false; + try + { + get(url); + assert(false, "Expected HttpException for refused connection"); + } + catch (HttpException e) + { + caught = true; + // Message should contain some curl-level error description + assert(e.msg.length > 0); + } + assert(caught, "Should have thrown HttpException"); +} + +// ── Test 5: GET with 500 status → HttpException ───────── +@safe unittest +{ + auto url = bindAndSpawn(httpResponse(500, "Internal Server Error", "boom")); + bool caught = false; + try + { + get(url); + assert(false, "Expected HttpException for 500"); + } + catch (HttpException e) + { + caught = true; + assert(e.msg.canFind("HTTP 500"), + "Message should contain 'HTTP 500', got: " ~ e.msg); + assert(e.msg.canFind(url), + "Message should contain url, got: " ~ e.msg); + } + assert(caught, "Should have thrown HttpException"); +}