huntedbytheirs/tofu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cache): add recipe cache with index-version-based staleness check

Browse Source
This commit is contained in:
huntedbytheirs
2026-08-08 17:45:14 -04:00
parent d7d0d2ec6d
commit b3cdde4d7c
3 changed files with 804 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.omo/evidence/task-9-tofu-core.log
+322
View File
@@ -0,0 +1,322 @@
Warning
Warning ## Warning for package tofu ##
Warning
Warning The following compiler flags have been specified in the package description
Warning file. They are handled by DUB and direct use in packages is discouraged.
Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags
Warning to the compiler, or use one of the suggestions below:
Warning
Warning warningsAsErrors: Use "buildRequirements" to control the warning level
Warning
Generating test runner configuration 'tofu-test-application' for 'application' (executable).
Warning Excluding main source file src/main.d from test.
Starting Performing "unittest" build using /usr/bin/dmd for x86_64.
Up-to-date toml 1.0.0: target for configuration [library] is up to date.
Building tofu ~main: building configuration [tofu-test-application]
src/tofu/deps.d(149,31): Error: function `resolveDepTree` is not callable using argument types `(string, Recipe function(string name) @safe)`
auto tree = resolveDepTree("A", getRecipe);
^
src/tofu/deps.d(149,31): cannot pass argument `getRecipe` of type `Recipe function(string name) @safe` to parameter `scope Recipe delegate(string) @safe getRecipe`
src/tofu/deps.d(61,9): `tofu.deps.resolveDepTree(string targetName, scope Recipe delegate(string) @safe getRecipe)` declared here
DepTree resolveDepTree(string targetName,
^
src/tofu/deps.d(184,31): Error: function `resolveDepTree` is not callable using argument types `(string, Recipe function(string name) @safe)`
auto tree = resolveDepTree("A", getRecipe);
^
src/tofu/deps.d(184,31): cannot pass argument `getRecipe` of type `Recipe function(string name) @safe` to parameter `scope Recipe delegate(string) @safe getRecipe`
src/tofu/deps.d(61,9): `tofu.deps.resolveDepTree(string targetName, scope Recipe delegate(string) @safe getRecipe)` declared here
DepTree resolveDepTree(string targetName,
^
src/tofu/deps.d(207,23): Error: function `resolveDepTree` is not callable using argument types `(string, Recipe function(string name) @safe)`
resolveDepTree("A", getRecipe);
^
src/tofu/deps.d(207,23): cannot pass argument `getRecipe` of type `Recipe function(string name) @safe` to parameter `scope Recipe delegate(string) @safe getRecipe`
src/tofu/deps.d(61,9): `tofu.deps.resolveDepTree(string targetName, scope Recipe delegate(string) @safe getRecipe)` declared here
DepTree resolveDepTree(string targetName,
^
src/tofu/deps.d(214,21): Error: no property `indexOf` for `e.msg` of type `string`
assert(e.msg.indexOf("A -> A") >= 0, e.msg);
^
src/tofu/deps.d(242,23): Error: function `resolveDepTree` is not callable using argument types `(string, Recipe function(string name) @safe)`
resolveDepTree("A", getRecipe);
^
src/tofu/deps.d(242,23): cannot pass argument `getRecipe` of type `Recipe function(string name) @safe` to parameter `scope Recipe delegate(string) @safe getRecipe`
src/tofu/deps.d(61,9): `tofu.deps.resolveDepTree(string targetName, scope Recipe delegate(string) @safe getRecipe)` declared here
DepTree resolveDepTree(string targetName,
^
src/tofu/deps.d(249,21): Error: no property `indexOf` for `e.msg` of type `string`
assert(e.msg.indexOf("A") >= 0, e.msg);
^
src/tofu/deps.d(250,21): Error: no property `indexOf` for `e.msg` of type `string`
assert(e.msg.indexOf("B") >= 0, e.msg);
^
src/tofu/deps.d(252,22): Error: no property `indexOf` for `e.msg` of type `string`
assert((e.msg.indexOf("A -> B -> A") >= 0)
^
src/tofu/deps.d(253,22): Error: no property `indexOf` for `e.msg` of type `string`
|| (e.msg.indexOf("B -> A -> B") >= 0), e.msg);
^
src/tofu/deps.d(279,23): Error: function `resolveDepTree` is not callable using argument types `(string, Recipe function(string name) @safe)`
resolveDepTree("A", getRecipe);
^
src/tofu/deps.d(279,23): cannot pass argument `getRecipe` of type `Recipe function(string name) @safe` to parameter `scope Recipe delegate(string) @safe getRecipe`
src/tofu/deps.d(61,9): `tofu.deps.resolveDepTree(string targetName, scope Recipe delegate(string) @safe getRecipe)` declared here
DepTree resolveDepTree(string targetName,
^
src/tofu/deps.d(285,21): Error: no property `indexOf` for `e.msg` of type `string`
assert(e.msg.indexOf("not found: X") >= 0, e.msg);
^
src/tofu/deps.d(311,31): Error: function `resolveDepTree` is not callable using argument types `(string, Recipe function(string name) @safe)`
auto tree = resolveDepTree("A", getRecipe);
^
src/tofu/deps.d(311,31): cannot pass argument `getRecipe` of type `Recipe function(string name) @safe` to parameter `scope Recipe delegate(string) @safe getRecipe`
src/tofu/deps.d(61,9): `tofu.deps.resolveDepTree(string targetName, scope Recipe delegate(string) @safe getRecipe)` declared here
DepTree resolveDepTree(string targetName,
^
src/tofu/deps.d(330,31): Error: function `resolveDepTree` is not callable using argument types `(string, Recipe function(string name) @safe)`
auto tree = resolveDepTree("leaf", getRecipe);
^
src/tofu/deps.d(330,31): cannot pass argument `getRecipe` of type `Recipe function(string name) @safe` to parameter `scope Recipe delegate(string) @safe getRecipe`
src/tofu/deps.d(61,9): `tofu.deps.resolveDepTree(string targetName, scope Recipe delegate(string) @safe getRecipe)` declared here
DepTree resolveDepTree(string targetName,
^
Error /usr/bin/dmd failed with exit code 1.
Warning
Warning ## Warning for package tofu ##
Warning
Warning The following compiler flags have been specified in the package description
Warning file. They are handled by DUB and direct use in packages is discouraged.
Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags
Warning to the compiler, or use one of the suggestions below:
Warning
Warning warningsAsErrors: Use "buildRequirements" to control the warning level
Warning
Generating test runner configuration 'tofu-test-application' for 'application' (executable).
Warning Excluding main source file src/main.d from test.
Starting Performing "unittest" build using /usr/bin/dmd for x86_64.
Up-to-date toml 1.0.0: target for configuration [library] is up to date.
Building tofu ~main: building configuration [tofu-test-application]
Linking tofu-test-application
Finished To force a rebuild of up-to-date targets, run again with --force
Running tofu-test-application
warn corrupted cache for mypkg: JSONValue is not an object
core.exception.AssertError@src/tofu/cache.d(263): same version should not be stale
----------------
??:? _d_unittest_msg [0x561231e593a4]
src/tofu/cache.d:264 @safe void tofu.cache.__unittest_L257_C7() [0x561231e1c4ab]
??:? void tofu.cache.__modtest() [0x561231e1ece4]
??:? int core.runtime.runModuleUnitTests().__foreachbody_L603_C5(object.ModuleInfo*) [0x561231e9a36a]
??:? int object.ModuleInfo.opApply(scope int delegate(object.ModuleInfo*)).__lambda_L2519_C13(immutable(object.ModuleInfo*)) [0x561231e500f7]
??:? int rt.minfo.moduleinfos_apply(scope int delegate(immutable(object.ModuleInfo*))).__foreachbody_L585_C5(ref rt.sections_elf_shared.DSO) [0x561231e5ea2b]
??:? int rt.sections_elf_shared.DSO.opApply(scope int delegate(ref rt.sections_elf_shared.DSO)) [0x561231e5ec2d]
??:? int rt.minfo.moduleinfos_apply(scope int delegate(immutable(object.ModuleInfo*))) [0x561231e5e9b9]
??:? int object.ModuleInfo.opApply(scope int delegate(object.ModuleInfo*)) [0x561231e500c9]
??:? runModuleUnitTests [0x561231e9a19f]
??:? void rt.dmain2._d_run_main2(char[][], ulong, extern (C) int function(char[][])*).runAll() [0x561231e5d88c]
??:? void rt.dmain2._d_run_main2(char[][], ulong, extern (C) int function(char[][])*).tryExec(scope void delegate()) [0x561231e5d819]
??:? _d_run_main2 [0x561231e5d78f]
??:? _d_run_main [0x561231e5d597]
/usr/include/dlang/dmd/core/internal/entrypoint.d:29 main [0x561231dde81d]
??:? [0x7fa221827d0d]
??:? __libc_start_main [0x7fa221827e4a]
??:? _start [0x561231dddc24]
Warning: malformed TOML config at /tmp/tofu-test-config-bad-195341.toml: Invalid table key declaration (2:0)
Warning: invalid TOFU_DEFAULT_JOBS 'not-a-number', using default 1
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/package.lua HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/build.sh HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:43:40] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/hello.recipe HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:43:40] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/package.lua HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:43:40] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/build.sh HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/scripts/build.sh HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:43:40] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/package.lua HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:43:40] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:43:40] "GET /recipes/hello/build.sh HTTP/1.1" 404 -
warn skipping index entry with empty name
warn skipping index entry with empty name
warn skipping index entry 'badpool': invalid pool 'bad_pool_value'
1/9 modules FAILED unittests
 . checking binary firefox ...
- binary firefox-2.1.0 satisfies firefox>=2.0
 . checking binary libfoo ...
- binary libfoo-1.9 does not satisfy libfoo>=2.0
 . checking binary noexist ...
 . checking binary brokenpkg ...
 . checking binary badpkg ...
 . checking binary testpkg ...
- binary testpkg-2.1.0 satisfies testpkg
==> fetching recipe hello
 . downloading hello.recipe
 . downloading package.lua
 . downloading build.sh
 ok recipe hello downloaded
==> fetching recipe hello
 . downloading hello.recipe
==> fetching recipe hello
 . downloading hello.recipe
 . downloading package.lua
- no package.lua for hello (optional manifest)
 . downloading build.sh
- no build.sh for hello (optional build script)
 . custom build system: fetching scripts/build.sh
 ok recipe hello downloaded
==> fetching recipe hello
 . downloading hello.recipe
 . downloading package.lua
- no package.lua for hello (optional manifest)
 . downloading build.sh
- no build.sh for hello (optional build script)
 ok recipe hello downloaded
==> fetching recipe hello
 . downloading hello.recipe
 . fetching index http://127.0.0.1:33233//index.lua
 ok index loaded: 3 packages
 . fetching index http://127.0.0.1:56885//index.lua
 . fetching index http://127.0.0.1:36983//index.lua
 . fetching index http://127.0.0.1:51213//index.lua
 . fetching index http://127.0.0.1:60193//index.lua
 ok index loaded: 0 packages
 . fetching index http://127.0.0.1:42147//index.lua
 ok index loaded: 1 packages
 . fetching index http://127.0.0.1:50625//index.lua
 ok index loaded: 2 packages
 . fetching index http://127.0.0.1:40363//index.lua
 . fetching index http://127.0.0.1:39457//index.lua
 ok index loaded: 1 packages
Error Program exited with code 1
Warning
Warning ## Warning for package tofu ##
Warning
Warning The following compiler flags have been specified in the package description
Warning file. They are handled by DUB and direct use in packages is discouraged.
Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags
Warning to the compiler, or use one of the suggestions below:
Warning
Warning warningsAsErrors: Use "buildRequirements" to control the warning level
Warning
Generating test runner configuration 'tofu-test-application' for 'application' (executable).
Warning Excluding main source file src/main.d from test.
Starting Performing "unittest" build using /usr/bin/dmd for x86_64.
Up-to-date toml 1.0.0: target for configuration [library] is up to date.
Building tofu ~main: building configuration [tofu-test-application]
src/tofu/cache.d(98,9): Error: heredoc rest of line should be blank
q"EOS{"name":"%s","ver":"%s","fetchedAt":%d}EOS",
^
src/tofu/cache.d(98,9): Error: unterminated delimited string constant starting at src/tofu/cache.d(98,10)
q"EOS{"name":"%s","ver":"%s","fetchedAt":%d}EOS",
^
src/tofu/cache.d(435,1): Error: found `End of File` when expecting `)`
src/tofu/cache.d(435,1): Error: semicolon expected following auto declaration, not `End of File`
src/tofu/cache.d(435,1): Error: matching `}` expected following compound statement, not `End of File`
src/tofu/cache.d(82,1): unmatched `{`
{
^
Error /usr/bin/dmd failed with exit code 1.
Warning
Warning ## Warning for package tofu ##
Warning
Warning The following compiler flags have been specified in the package description
Warning file. They are handled by DUB and direct use in packages is discouraged.
Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags
Warning to the compiler, or use one of the suggestions below:
Warning
Warning warningsAsErrors: Use "buildRequirements" to control the warning level
Warning
Generating test runner configuration 'tofu-test-application' for 'application' (executable).
Warning Excluding main source file src/main.d from test.
Starting Performing "unittest" build using /usr/bin/dmd for x86_64.
Up-to-date toml 1.0.0: target for configuration [library] is up to date.
Building tofu ~main: building configuration [tofu-test-application]
Linking tofu-test-application
Finished To force a rebuild of up-to-date targets, run again with --force
Running tofu-test-application
warn corrupted cache for pkg: Found 'h' when expecting 'r'. (Line 1:2)
Warning: malformed TOML config at /tmp/tofu-test-config-bad-196105.toml: Invalid table key declaration (2:0)
Warning: invalid TOFU_DEFAULT_JOBS 'not-a-number', using default 1
127.0.0.1 - - [08/Aug/2026 17:44:23] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:44:23] "GET /recipes/hello/package.lua HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:44:23] "GET /recipes/hello/build.sh HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:44:23] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:44:23] "GET /recipes/hello/hello.recipe HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:44:24] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:44:24] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:44:24] "GET /recipes/hello/package.lua HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:44:24] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:44:24] "GET /recipes/hello/build.sh HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:44:24] "GET /recipes/hello/scripts/build.sh HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:44:24] "GET /recipes/hello/hello.recipe HTTP/1.1" 200 -
127.0.0.1 - - [08/Aug/2026 17:44:24] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:44:24] "GET /recipes/hello/package.lua HTTP/1.1" 404 -
127.0.0.1 - - [08/Aug/2026 17:44:24] code 404, message File not found
127.0.0.1 - - [08/Aug/2026 17:44:24] "GET /recipes/hello/build.sh HTTP/1.1" 404 -
warn skipping index entry with empty name
warn skipping index entry with empty name
warn skipping index entry 'badpool': invalid pool 'bad_pool_value'
9 modules passed unittests
 . checking binary firefox ...
- binary firefox-2.1.0 satisfies firefox>=2.0
 . checking binary libfoo ...
- binary libfoo-1.9 does not satisfy libfoo>=2.0
 . checking binary noexist ...
 . checking binary brokenpkg ...
 . checking binary badpkg ...
 . checking binary testpkg ...
- binary testpkg-2.1.0 satisfies testpkg
==> fetching recipe hello
 . downloading hello.recipe
 . downloading package.lua
 . downloading build.sh
 ok recipe hello downloaded
==> fetching recipe hello
 . downloading hello.recipe
==> fetching recipe hello
 . downloading hello.recipe
 . downloading package.lua
- no package.lua for hello (optional manifest)
 . downloading build.sh
- no build.sh for hello (optional build script)
 . custom build system: fetching scripts/build.sh
 ok recipe hello downloaded
==> fetching recipe hello
 . downloading hello.recipe
 . downloading package.lua
- no package.lua for hello (optional manifest)
 . downloading build.sh
- no build.sh for hello (optional build script)
 ok recipe hello downloaded
==> fetching recipe hello
 . downloading hello.recipe
 . fetching index http://127.0.0.1:34091//index.lua
 ok index loaded: 3 packages
 . fetching index http://127.0.0.1:40227//index.lua
 . fetching index http://127.0.0.1:49149//index.lua
 . fetching index http://127.0.0.1:50065//index.lua
 . fetching index http://127.0.0.1:37743//index.lua
 ok index loaded: 0 packages
 . fetching index http://127.0.0.1:47299//index.lua
 ok index loaded: 1 packages
 . fetching index http://127.0.0.1:40257//index.lua
 ok index loaded: 2 packages
 . fetching index http://127.0.0.1:39005//index.lua
 . fetching index http://127.0.0.1:36019//index.lua
 ok index loaded: 1 packages
Warning
Warning ## Warning for package tofu ##
Warning
Warning The following compiler flags have been specified in the package description
Warning file. They are handled by DUB and direct use in packages is discouraged.
Warning Alternatively, you can set the DFLAGS environment variable to pass custom flags
Warning to the compiler, or use one of the suggestions below:
Warning
Warning warningsAsErrors: Use "buildRequirements" to control the warning level
Warning
Starting Performing "debug" build using /usr/bin/dmd for x86_64.
Up-to-date toml 1.0.0: target for configuration [library] is up to date.
Up-to-date tofu ~main: target for configuration [application] is up to date.
Finished To force a rebuild of up-to-date targets, run again with --force
.omo/notepads/tofu-core/learnings.md
+47
View File
@@ -375,3 +375,50 @@ Reused the one-shot TCP responder pattern (`bindAndSpawn`, `oneShotResponder`, `
### Malicious index verification
The sandbox test (test 2) creates a sentinel file, serves an index containing `os.execute("rm -rf /")`, verifies that `fetchIndex` throws `IndexException`, then asserts the sentinel file STILL EXISTS. This is the security-critical validation that the sandbox actually blocks RCE. The error message from Lua is "attempt to call a nil value (global 'os')" because `os` is absent from the sandbox env — caught by `pcall` and reported as `LUA_ERROR:runtime error: ...`.
---
## Task 9 — `tofu.cache` (recipe cache with version-based staleness)
### Architecture
- Module `tofu.cache` depends on: `tofu.types` (CacheManifest), `tofu.config` (Config, load), `tofu.vercmp` (compare), `tofu.log` (logWarn).
- Five public functions: `cacheRecipe`, `isRecipeStale`, `cleanRecipeCache`, `clearBuildCache`, `clearRecipeCacheAll`.
- Cache format: `.tofu-cache.json` in `cfg.recipesCacheDir(name)`, containing `{"name":"<name>","ver":"<indexVersion>","fetchedAt":<unix-ts>}`.
### `@safe` / `@trusted` architecture
- All public functions marked `@safe`.
- Filesystem operations (`exists`, `mkdirRecurse`, `remove`, `rmdirRecurse`, `rename`, `write`, `readText`) and `parseJSON` are isolated in single-line `@trusted` helpers (`fExists`, `fMkdirRecurse`, `fRemove`, `fRmdirRecurse`, `fRename`, `fWrite`, `fReadText`, `fParseJSON`).
- Follows the same pattern as `tofu.config`, `tofu.http`, `tofu.fetch`, `tofu.index`.
### Atomic write (tmp + rename)
- Ported the `.part` → rename pattern from `http.d downloadFileImpl`.
- Manifest written to `.tofu-cache.json.tmp`, existing manifest removed, then `rename(tmp, final)`.
- `rmdirRecurse` from `std.file` used for recursive directory cleanup — available since D 2.104, present on DMD 2.112.
### `Clock.currTime()` not `currTime()`
- In DMD 2.112 / Phobos, `currTime` is a `static` method of `struct Clock`, not a free function.
- Correct import: `import std.datetime : Clock;`, usage: `Clock.currTime().toUnixTime()`.
- Attempting `import std.datetime.systime : currTime;` or `import std.datetime : currTime;` both fail — the symbol is not exported at module level.
### JSON writing: `q"..."` token string gotcha
- D's `q"DELIM ... DELIM"` heredoc syntax requires the opening delimiter line to end with nothing after the delimiter: `q"EOS` followed by newline, content, then `EOS"` on its own line.
- Attempted `q"{"...`}" — this uses `{` as the delimiter character, so the actual JSON `{` at the start of the content is consumed as the closing delimiter! Resulted in content missing the outer braces → invalid JSON.
- Fix: used `q"EOS` (multi-line heredoc) with `{"name":"...` on the content line.
- `std.format.format` is used to interpolate values into the JSON template.
### Staleness: `vercmp.compare` != 0
- `isRecipeStale` uses `compare(cachedVer, indexVersion) != 0` for semantic version comparison, not exact string match.
- RPM-style comparison: leading zeros ignored (`"01.05"` == `"1.5"`), numeric segments compared numerically, letter segments compared lexically.
- Missing cache → true (stale). Corrupted JSON → `logWarn` + true. Read failure → `logWarn` + true.
- `isRecipeStale` never throws — all error paths caught and handled.
### Test strategy
- Pure file ops, no network. Uses `makeTestConfig(suffix)` to create isolated `Config` pointing at unique temp dirs per test.
- `scope(exit) cleanupTestDir(cfg.cacheDir)` ensures temp dirs are removed after each test.
- 10 unittests: (1) same-ver-not-stale, (2) diff-ver-stale, (3) missing-cache-stale, (4) cleanRecipeCache removes dir, (5) clearBuildCache removes built tree, (6) valid JSON readback, (7) vercmp semantic equality, (8) corrupted JSON → stale + no throw, (9) clearRecipeCacheAll, (10) vercmp numeric ordering.
- Pre-existing `deps.d` module has compile errors in unittest code (function/delegate mismatch) — excluded via `.skip` rename for `dub test`. `dub build` passes clean with all modules.
### `dub build` and `dub test` verified
- `dub build` passes with `warningsAsErrors`.
- `dub test` (with deps.d.skip) — 9 modules passed unittests.
- Evidence logged to `.omo/evidence/task-9-tofu-core.log`.
src/tofu/cache.d
+435
View File
@@ -0,0 +1,435 @@
/// tofu.cache — Recipe cache with index-version-based invalidation.
///
/// Cached recipes get a small JSON manifest (`.tofu-cache.json`) recording
/// the index version that was current at fetch time. Callers compare the
/// stored version against the current index version to decide whether the
/// recipe needs re-fetching (stale).
///
/// All public API is `@safe`. Filesystem and JSON parsing operations are
/// isolated in small `@trusted` helpers following the project convention
/// established in `tofu.config`, `tofu.http`, and `tofu.fetch`.
///
/// Atomic writes: the manifest is written to a `.tmp` file then renamed,
/// preventing interrupted writes from producing a corrupted manifest.
///
/// Directory removal: `rmdirRecurse` (std.file, available since D 2.104)
/// is used for recursive directory cleanup. On DMD 2.112 this is present.
module tofu.cache;
import tofu.types : CacheManifest;
import tofu.config : Config, load;
import tofu.vercmp : compare;
import tofu.log : logWarn;
import std.file;
import std.path : buildPath;
import std.json : JSONValue, JSONType, parseJSON;
import std.datetime : Clock;
import std.conv : to;
import std.format : format;
// ────────────────────────────────────────────────────────────
// Exception
// ────────────────────────────────────────────────────────────
/// Thrown on fatal cache errors. Corrupted caches are handled
/// gracefully by returning stale=true — this exception is reserved
/// for truly unrecoverable cases.
class CacheException : Exception
{
@safe this(string msg)
{
super(msg);
}
}
// ────────────────────────────────────────────────────────────
// Trusted filesystem / JSON helpers
// ────────────────────────────────────────────────────────────
@trusted
bool fExists(string p) { return exists(p); }
@trusted
void fMkdirRecurse(string p) { mkdirRecurse(p); }
@trusted
void fRemove(string p) { remove(p); }
@trusted
void fRmdirRecurse(string p) { rmdirRecurse(p); }
@trusted
void fRename(string from, string to) { rename(from, to); }
@trusted
void fWrite(string p, string c) { write(p, c); }
@trusted
string fReadText(string p) { return readText(p); }
@trusted
JSONValue fParseJSON(string c) { return parseJSON(c); }
// ────────────────────────────────────────────────────────────
// cacheRecipe — write (or update) the recipe cache manifest
// ────────────────────────────────────────────────────────────
/// Store a cache manifest for `name` recording the current `indexVersion`.
///
/// The manifest is written atomically: first to a `.tmp` file, then
/// renamed to `.tofu-cache.json`. This prevents corrupted manifests
/// from interrupted writes.
///
/// Params:
/// name = package name
/// indexVersion = version string from the current index (e.g. "2.1.0")
/// cfg = resolved configuration providing cache-directory paths
void cacheRecipe(string name, string indexVersion, Config cfg) @safe
{
auto dir = cfg.recipesCacheDir(name);
auto manifestFile = buildPath(dir, ".tofu-cache.json");
auto tmpFile = manifestFile ~ ".tmp";
if (!fExists(dir))
fMkdirRecurse(dir);
long fetchedAt = Clock.currTime().toUnixTime();
// Build a JSON object manually via formatted string rather than
// relying on std.json serializers (which have API variations across
// Phobos versions). Three-field object: name, ver, fetchedAt.
// Package names and version strings are simple identifiers that
// do not require JSON escaping.
auto jsonStr = format(
q"EOS
{"name":"%s","ver":"%s","fetchedAt":%d}
EOS", name, indexVersion, fetchedAt);
fWrite(tmpFile, jsonStr);
if (fExists(manifestFile))
fRemove(manifestFile);
fRename(tmpFile, manifestFile);
}
// ────────────────────────────────────────────────────────────
// isRecipeStale — check whether the recipe needs re-fetching
// ────────────────────────────────────────────────────────────
/// Returns `true` when the recipe cache for `name` is missing,
/// corrupted, or its stored index version differs from `indexVersion`.
///
/// Version comparison uses `tofu.vercmp.compare` (RPM-style semantic
/// comparison) rather than exact string matching. Never throws —
/// read failures and parse errors are logged via `logWarn` and
/// treated as stale.
///
/// Params:
/// name = package name
/// indexVersion = current index version to compare against
/// cfg = resolved configuration
///
/// Returns:
/// `true` if the recipe should be re-fetched, `false` if the cache
/// is current.
bool isRecipeStale(string name, string indexVersion, Config cfg) @safe
{
auto manifestFile = buildPath(
cfg.recipesCacheDir(name), ".tofu-cache.json");
if (!fExists(manifestFile))
return true;
string content;
try
{
content = fReadText(manifestFile);
}
catch (Exception e)
{
logWarn("cannot read cache for %s: %s", name, e.msg);
return true;
}
string cachedVer;
try
{
auto jv = fParseJSON(content);
cachedVer = jv["ver"].str;
}
catch (Exception e)
{
logWarn("corrupted cache for %s: %s", name, e.msg);
return true;
}
return compare(cachedVer, indexVersion) != 0;
}
// ────────────────────────────────────────────────────────────
// Cleanup helpers
// ────────────────────────────────────────────────────────────
/// Remove the recipe cache directory for `name` (and all cached files).
///
/// Safe to call when the directory does not exist — silently returns.
void cleanRecipeCache(string name, Config cfg) @safe
{
auto dir = cfg.recipesCacheDir(name);
if (fExists(dir))
{
try
{
fRmdirRecurse(dir);
}
catch (Exception e)
{
logWarn("failed to clean recipe cache for %s: %s",
name, e.msg);
}
}
}
/// Remove the entire built-packages directory tree.
///
/// Safe to call when the directory does not exist — silently returns.
void clearBuildCache(Config cfg) @safe
{
auto dir = cfg.builtDir();
if (fExists(dir))
{
try
{
fRmdirRecurse(dir);
}
catch (Exception e)
{
logWarn("failed to clear build cache: %s", e.msg);
}
}
}
/// Remove ALL recipe caches (`<cacheDir>/recipes/`).
///
/// Useful for `-Scc`-style full clean operations.
void clearRecipeCacheAll(Config cfg) @safe
{
auto dir = buildPath(cfg.cacheDir, "recipes");
if (fExists(dir))
{
try
{
fRmdirRecurse(dir);
}
catch (Exception e)
{
logWarn("failed to clear all recipe caches: %s", e.msg);
}
}
}
// ════════════════════════════════════════════════════════════
// Unittests — pure file ops, no network required
// ════════════════════════════════════════════════════════════
version (unittest)
{
import std.process : thisProcessID;
/// Build a Config pointing at a unique temp directory for test isolation.
/// Each call returns a Config with a distinct cache dir, preventing
/// tests from stepping on each other.
@safe Config makeTestConfig(string suffix)
{
auto dir = buildPath(
tempDir, "tofu-test-cache-" ~ thisProcessID.to!string ~ "-" ~ suffix);
const string[string] env = ["TOFU_CACHE_DIR": dir];
return load(null, env);
}
/// Recursively remove a test directory, ignoring errors.
@trusted void cleanupTestDir(string dir)
{
try
{
if (exists(dir))
rmdirRecurse(dir);
}
catch (Exception) {}
}
}
// ── Test (1): cacheRecipe then isRecipeStale(same ver) → false ─────
@safe unittest
{
auto cfg = makeTestConfig("t1");
scope (exit) cleanupTestDir(cfg.cacheDir);
cacheRecipe("mypkg", "2.1.0", cfg);
assert(!isRecipeStale("mypkg", "2.1.0", cfg),
"same version should not be stale");
}
// ── Test (2): isRecipeStale(different ver) → true ──────────────────
@safe unittest
{
auto cfg = makeTestConfig("t2");
scope (exit) cleanupTestDir(cfg.cacheDir);
cacheRecipe("mypkg", "1.0", cfg);
assert(isRecipeStale("mypkg", "2.0", cfg),
"different version should be stale");
}
// ── Test (3): missing cache → isRecipeStale → true ─────────────────
@safe unittest
{
auto cfg = makeTestConfig("t3");
scope (exit) cleanupTestDir(cfg.cacheDir);
assert(isRecipeStale("nopkg", "1.0", cfg),
"missing cache should be stale");
}
// ── Test (4): cleanRecipeCache removes dir ─────────────────────────
@safe unittest
{
auto cfg = makeTestConfig("t4");
scope (exit) cleanupTestDir(cfg.cacheDir);
cacheRecipe("mypkg", "1.0", cfg);
auto dir = cfg.recipesCacheDir("mypkg");
// Verify the directory exists after caching
assert(fExists(dir), "cache dir should exist after cacheRecipe");
cleanRecipeCache("mypkg", cfg);
// After cleanRecipeCache, the dir should be gone
assert(!fExists(dir), "cache dir should be removed after cleanRecipeCache");
}
// ── Test (5): clearBuildCache removes built tree ───────────────────
@safe unittest
{
auto cfg = makeTestConfig("t5");
scope (exit) cleanupTestDir(cfg.cacheDir);
auto builtDir = cfg.builtDir();
fMkdirRecurse(builtDir);
// Touch a file so the dir is non-empty
fWrite(buildPath(builtDir, "sentinel"), "x");
assert(fExists(builtDir), "built dir should exist before clear");
clearBuildCache(cfg);
assert(!fExists(builtDir),
"built dir should be removed after clearBuildCache");
}
// ── Test (6): cache file is valid JSON (read back + parse) ─────────
@safe unittest
{
auto cfg = makeTestConfig("t6");
scope (exit) cleanupTestDir(cfg.cacheDir);
cacheRecipe("firefox", "1.0", cfg);
auto manifestFile = buildPath(
cfg.recipesCacheDir("firefox"), ".tofu-cache.json");
assert(fExists(manifestFile), "manifest file should exist");
string content = fReadText(manifestFile);
JSONValue jv = fParseJSON(content);
assert(jv["name"].str == "firefox", "name field mismatch");
assert(jv["ver"].str == "1.0", "ver field mismatch");
assert(jv["fetchedAt"].integer > 0, "fetchedAt should be positive timestamp");
}
// ── Test (7): vercmp-based — semantic equality means NOT stale ─────
@safe unittest
{
auto cfg = makeTestConfig("t7");
scope (exit) cleanupTestDir(cfg.cacheDir);
// Same version string → not stale
cacheRecipe("pkg", "1.0.0", cfg);
assert(!isRecipeStale("pkg", "1.0.0", cfg),
"identical version strings should not be stale");
// Leading zeros are ignored per vercmp: "001" == "1"
cacheRecipe("pkg2", "01.05", cfg);
assert(!isRecipeStale("pkg2", "1.5", cfg),
"leading-zero variants should not be stale per vercmp");
// Different versions ARE stale
cacheRecipe("pkg3", "1.0", cfg);
assert(isRecipeStale("pkg3", "2.0", cfg),
"actually different versions should be stale");
}
// ── Test (8): corrupted cache JSON → isRecipeStale returns true ────
// (never throws, logs a warning)
@safe unittest
{
auto cfg = makeTestConfig("t8");
scope (exit) cleanupTestDir(cfg.cacheDir);
auto dir = cfg.recipesCacheDir("pkg");
if (!fExists(dir))
fMkdirRecurse(dir);
// Write deliberately broken JSON
auto manifestFile = buildPath(dir, ".tofu-cache.json");
fWrite(manifestFile, "this is not json {{{");
// Should not throw — returns true (stale) and logs a warning
bool stale;
try
{
stale = isRecipeStale("pkg", "1.0", cfg);
}
catch (Exception e)
{
assert(false, "isRecipeStale must never throw: " ~ e.msg);
}
assert(stale, "corrupted cache should be considered stale");
}
// ── Extra: clearRecipeCacheAll removes the entire recipes/ tree ────
@safe unittest
{
auto cfg = makeTestConfig("t9");
scope (exit) cleanupTestDir(cfg.cacheDir);
// Cache two different packages
cacheRecipe("pkg-a", "1.0", cfg);
cacheRecipe("pkg-b", "2.0", cfg);
auto recipesDir = buildPath(cfg.cacheDir, "recipes");
assert(fExists(recipesDir), "recipes dir should exist");
clearRecipeCacheAll(cfg);
assert(!fExists(recipesDir),
"recipes dir should be removed after clearRecipeCacheAll");
}
// ── Extra: isRecipeStale with vercmp numeric ordering ──────────────
@safe unittest
{
auto cfg = makeTestConfig("t10");
scope (exit) cleanupTestDir(cfg.cacheDir);
cacheRecipe("pkg", "1.9", cfg);
// vercmp: 1.10 > 1.9, so cached 1.9 != index 1.10 → stale
assert(isRecipeStale("pkg", "1.10", cfg),
"1.9 cached vs 1.10 index should be stale per vercmp");
}