feat: add index-aware recipe resolution with repo support

- fetch_recipe_from_repos() iterates repos by priority, channels, with
  mirror fallback — caches indexes at /cache/indexes/

- Conditional HTTP downloads (curl -z) to only re-fetch stale indexes

- build_registry and fetch-package prefer repos { } over remotes = [...]
  when both are present

- Fallback chain: local .kap → repos (index-aware) → remotes (legacy)
This commit is contained in:
2026-08-04 13:50:29 -04:00
parent 4f1ce17ec8
commit 7b0538c392
3 changed files with 267 additions and 2 deletions
+7
View File
@@ -1,5 +1,7 @@
#pragma once #pragma once
#include "kappa/dsl/system.hpp"
#include <string> #include <string>
#include <vector> #include <vector>
@@ -13,7 +15,12 @@ struct RecipeResult {
std::string error; std::string error;
}; };
// Legacy: flat URL list (backward compat with remotes = [...])
RecipeResult fetch_recipe(const std::string& name, RecipeResult fetch_recipe(const std::string& name,
const std::vector<std::string>& remotes); const std::vector<std::string>& remotes);
// Index-aware: named repos with channels, mirrors, and cached indexes
RecipeResult fetch_recipe_from_repos(const std::string& name,
const std::vector<dsl::RepoDef>& repos);
} // namespace kappa::fetch } // namespace kappa::fetch
+232
View File
@@ -125,4 +125,236 @@ RecipeResult fetch_recipe(const std::string& name,
return result; return result;
} }
// ---------------------------------------------------------------------------
// Index-aware recipe fetching
// ---------------------------------------------------------------------------
static std::string sanitize_filename(const std::string& s) {
std::string out = s;
for (auto& c : out) {
if (c == '/' || c == '\\' || c == ':') c = '_';
}
return out;
}
static int exec_curl_conditional(const std::string& output_path,
const std::string& url,
const std::string& time_cond_path) {
std::vector<std::string> argv = {"curl", "-Lsf", "-o", output_path};
if (!time_cond_path.empty()) {
argv.push_back("-z");
argv.push_back(time_cond_path);
}
argv.push_back(url);
std::vector<std::vector<char>> argv_storage(argv.size());
std::vector<char*> cargs;
for (size_t i = 0; i < argv.size(); ++i) {
argv_storage[i].assign(argv[i].begin(), argv[i].end());
argv_storage[i].push_back('\0');
cargs.push_back(argv_storage[i].data());
}
cargs.push_back(nullptr);
pid_t pid = fork();
if (pid == 0) {
execvp(cargs[0], cargs.data());
_exit(127);
}
if (pid < 0) return -1;
int status = 0;
while (waitpid(pid, &status, 0) == -1 && errno == EINTR) {}
return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
}
static std::optional<dsl::IndexDef> load_cached_index(
const std::string& repo_name,
const std::string& channel) {
auto key = sanitize_filename(repo_name + "-" + channel) + ".kap";
auto path = paths::cache_dir() / "indexes" / key;
if (!std::filesystem::exists(path)) return std::nullopt;
std::ifstream in(path);
if (!in) return std::nullopt;
std::ostringstream buf;
buf << in.rdbuf();
try {
return dsl::parse_index(buf.str());
} catch (...) {
return std::nullopt;
}
}
static bool download_index(const dsl::RepoDef& repo,
const std::string& channel,
const std::filesystem::path& cache_path) {
auto url = repo.url;
if (!url.empty() && url.back() != '/') url += '/';
url += channel + "/index.kap";
auto temp = cache_path.string() + ".tmp";
// Try primary URL
int rc = exec_curl_conditional(temp, url, cache_path.string());
if (rc != 0) {
// Try mirrors
for (auto& mirror : repo.mirrors) {
auto murl = mirror;
if (!murl.empty() && murl.back() != '/') murl += '/';
murl += channel + "/index.kap";
rc = exec_curl_conditional(temp, murl, "");
if (rc == 0) break;
}
}
if (rc != 0) return false;
// If curl returned 0 but the file might be empty (304 not modified),
// move temp to cache only if it has content
std::error_code ec;
if (std::filesystem::file_size(temp, ec) > 0 || ec) {
std::filesystem::rename(temp, cache_path, ec);
return !ec;
}
// 304: no change, remove temp
std::filesystem::remove(temp, ec);
return true; // cache is still valid
}
static std::optional<std::string> download_recipe(
const std::string& name,
const dsl::RepoDef& repo,
const std::string& channel) {
auto remote_path = channel + "/" + name + ".kap";
auto temp = paths::temp_dir() / (name + ".kap.tmp");
// Build URL list: primary + mirrors
std::vector<std::string> urls;
auto url = repo.url;
if (!url.empty() && url.back() != '/') url += '/';
urls.push_back(url + remote_path);
for (auto& mirror : repo.mirrors) {
auto murl = mirror;
if (!murl.empty() && murl.back() != '/') murl += '/';
urls.push_back(murl + remote_path);
}
for (auto& u : urls) {
int rc = exec_curl_conditional(temp.string(), u, "");
if (rc == 0) {
std::ifstream in(temp);
if (!in) { std::filesystem::remove(temp); continue; }
std::ostringstream buf;
buf << in.rdbuf();
in.close();
std::filesystem::remove(temp);
return buf.str();
}
}
return std::nullopt;
}
RecipeResult fetch_recipe_from_repos(
const std::string& name,
const std::vector<dsl::RepoDef>& repos) {
RecipeResult result;
// Check cached version first
auto cache_path = paths::packages_dir() / (name + ".kap");
std::string cached_version;
if (std::filesystem::exists(cache_path)) {
std::ifstream in(cache_path);
if (in) {
std::ostringstream buf;
buf << in.rdbuf();
try {
auto pkg = dsl::parse(buf.str());
cached_version = pkg.version;
} catch (...) {}
}
}
// Ensure indexes cache directory exists
std::error_code ec;
std::filesystem::create_directories(paths::cache_dir() / "indexes", ec);
// Sort repos by priority (descending)
std::vector<const dsl::RepoDef*> sorted;
for (auto& r : repos) sorted.push_back(&r);
std::sort(sorted.begin(), sorted.end(),
[](auto* a, auto* b) { return a->priority > b->priority; });
std::string best_content;
std::string best_version;
for (auto* repo : sorted) {
for (auto& channel : repo->channels) {
// Check/update the index for this repo+channel
auto idx_key = sanitize_filename(repo->name + "-" + channel);
auto idx_cache = paths::cache_dir() / "indexes" / (idx_key + ".kap");
// Download fresh index (conditional on cache)
if (!download_index(*repo, channel, idx_cache)) continue;
// Load and search the index
auto idx = load_cached_index(repo->name, channel);
if (!idx) continue;
bool found = false;
for (auto& entry : idx->packages) {
if (entry.name != name) continue;
found = true;
break;
}
if (!found) continue;
// Download the recipe
auto content = download_recipe(name, *repo, channel);
if (!content) continue;
// Parse the downloaded recipe to get version
std::string remote_version;
try {
auto pkg = dsl::parse(*content);
remote_version = pkg.version;
} catch (...) {
continue;
}
if (remote_version > best_version) {
best_version = remote_version;
best_content = std::move(*content);
}
}
}
if (best_content.empty()) {
if (!cached_version.empty()) {
result.ok = true;
result.path = cache_path.string();
result.version = cached_version;
return result;
}
result.error = "package '" + name + "' not found in any repo";
return result;
}
// Update cache if remote is newer
if (best_version > cached_version || cached_version.empty()) {
std::ofstream out(cache_path);
if (!out) {
result.error = "cannot write to cache";
return result;
}
out << best_content;
result.updated = true;
}
result.ok = true;
result.path = cache_path.string();
result.version = best_version;
return result;
}
} // namespace kappa::fetch } // namespace kappa::fetch
+28 -2
View File
@@ -132,7 +132,26 @@ static resolve::Registry build_registry(const dsl::SystemConfig& cfg) {
} }
} }
// If not found locally, try remotes // If not found locally, try repos (index-aware) first, then legacy remotes
if (!found && !cfg.repos.empty()) {
auto result = fetch::fetch_recipe_from_repos(pref.name, cfg.repos);
if (result.ok && !result.path.empty()) {
std::ifstream in(result.path);
if (in) {
std::ostringstream buf;
buf << in.rdbuf();
try {
auto pkg = dsl::parse(buf.str());
registry[pkg.name] = std::move(pkg);
found = true;
} catch (const std::exception& e) {
std::cerr << "warning: parse error in remote recipe " << pref.name << ": " << e.what() << "\n";
} catch (...) {
std::cerr << "warning: unknown parse error in remote recipe " << pref.name << "\n";
}
}
}
}
if (!found && !cfg.remotes.empty()) { if (!found && !cfg.remotes.empty()) {
auto result = fetch::fetch_recipe(pref.name, cfg.remotes); auto result = fetch::fetch_recipe(pref.name, cfg.remotes);
if (result.ok && !result.path.empty()) { if (result.ok && !result.path.empty()) {
@@ -308,19 +327,26 @@ int main(int argc, char* argv[]) {
} }
try { try {
std::vector<std::string> remotes; std::vector<std::string> remotes;
std::vector<dsl::RepoDef> repos;
auto config_path = std::filesystem::path(paths::system_dir()) / "config.kap"; auto config_path = std::filesystem::path(paths::system_dir()) / "config.kap";
if (std::filesystem::exists(config_path)) { if (std::filesystem::exists(config_path)) {
auto cfg_src = read_file(config_path.c_str()); auto cfg_src = read_file(config_path.c_str());
try { try {
auto cfg = dsl::parse_system_config(cfg_src); auto cfg = dsl::parse_system_config(cfg_src);
remotes = cfg.remotes; remotes = cfg.remotes;
repos = std::move(cfg.repos);
} catch (const std::exception& e) { } catch (const std::exception& e) {
std::cerr << "warning: config parse error: " << e.what() << "\n"; std::cerr << "warning: config parse error: " << e.what() << "\n";
} catch (...) { } catch (...) {
std::cerr << "warning: unknown config parse error\n"; std::cerr << "warning: unknown config parse error\n";
} }
} }
auto result = fetch::fetch_recipe(file_arg, remotes); fetch::RecipeResult result;
if (!repos.empty()) {
result = fetch::fetch_recipe_from_repos(file_arg, repos);
} else {
result = fetch::fetch_recipe(file_arg, remotes);
}
if (result.ok) { if (result.ok) {
if (result.updated) { if (result.updated) {
std::cout << "fetched " << file_arg << " " << result.version std::cout << "fetched " << file_arg << " " << result.version