Files
kappa/src/fetch/recipe.cpp
T
huntedbytheirs 7b0538c392 feat: add index-aware recipe resolution with repo support
- fetch_recipe_from_repos() iterates repos by priority, channels, with
  mirror fallback — caches indexes at /cache/indexes/

- Conditional HTTP downloads (curl -z) to only re-fetch stale indexes

- build_registry and fetch-package prefer repos { } over remotes = [...]
  when both are present

- Fallback chain: local .kap → repos (index-aware) → remotes (legacy)
2026-08-04 13:50:29 -04:00

361 lines
11 KiB
C++

#include "kappa/fetch/recipe.hpp"
#include "kappa/dsl/parser.hpp"
#include "kappa/dsl/system.hpp"
#include "kappa/paths.hpp"
#include <sys/wait.h>
#include <unistd.h>
#include <filesystem>
#include <format>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
namespace kappa::fetch {
namespace {
int exec_curl(const std::string& output_path, const std::string& url) {
pid_t pid = fork();
if (pid == 0) {
execlp("curl", "curl", "-Lsf", "-o", output_path.c_str(), url.c_str(), nullptr);
_exit(127);
}
if (pid < 0) return -1;
int status = 0;
while (waitpid(pid, &status, 0) == -1 && errno == EINTR) {}
return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
}
} // namespace
RecipeResult fetch_recipe(const std::string& name,
const std::vector<std::string>& remotes) {
RecipeResult result;
// Check cached version first
auto cache_path = paths::packages_dir() / (name + ".kap");
std::string cached_version;
if (std::filesystem::exists(cache_path)) {
std::ifstream in(cache_path);
if (in) {
std::ostringstream buf;
buf << in.rdbuf();
try {
auto pkg = dsl::parse(buf.str());
cached_version = pkg.version;
} catch (...) {
// Corrupt cache — will re-download
}
}
}
// Try each remote
std::string best_content;
std::string best_version;
std::string best_url;
for (auto& remote : remotes) {
auto url = remote;
if (!url.empty() && url.back() != '/') url += '/';
url += name + ".kap";
// Download to temp
auto temp_path = paths::temp_dir() / (name + ".kap.tmp");
int rc = exec_curl(temp_path.string(), url);
if (rc != 0) continue;
// Parse downloaded file
std::ifstream in(temp_path);
if (!in) { std::filesystem::remove(temp_path); continue; }
std::ostringstream buf;
buf << in.rdbuf();
in.close();
std::string remote_version;
try {
auto pkg = dsl::parse(buf.str());
remote_version = pkg.version;
} catch (...) {
std::filesystem::remove(temp_path);
continue;
}
// Compare versions — keep the best (highest)
// Simple string comparison for now; semver later
if (remote_version > best_version) {
best_version = remote_version;
best_content = buf.str();
best_url = url;
}
std::filesystem::remove(temp_path);
}
if (best_content.empty()) {
if (!cached_version.empty()) {
// No remote available but have cache
result.ok = true;
result.path = cache_path.string();
result.version = cached_version;
result.updated = false;
return result;
}
result.error = "package '" + name + "' not found in any remote";
return result;
}
// Update cache if remote is newer
if (best_version > cached_version || cached_version.empty()) {
std::error_code ec;
std::filesystem::create_directories(paths::packages_dir(), ec);
std::ofstream out(cache_path);
if (!out) {
result.error = "cannot write to cache";
return result;
}
out << best_content;
result.updated = true;
}
result.ok = true;
result.path = cache_path.string();
result.version = best_version;
return result;
}
// ---------------------------------------------------------------------------
// Index-aware recipe fetching
// ---------------------------------------------------------------------------
static std::string sanitize_filename(const std::string& s) {
std::string out = s;
for (auto& c : out) {
if (c == '/' || c == '\\' || c == ':') c = '_';
}
return out;
}
static int exec_curl_conditional(const std::string& output_path,
const std::string& url,
const std::string& time_cond_path) {
std::vector<std::string> argv = {"curl", "-Lsf", "-o", output_path};
if (!time_cond_path.empty()) {
argv.push_back("-z");
argv.push_back(time_cond_path);
}
argv.push_back(url);
std::vector<std::vector<char>> argv_storage(argv.size());
std::vector<char*> cargs;
for (size_t i = 0; i < argv.size(); ++i) {
argv_storage[i].assign(argv[i].begin(), argv[i].end());
argv_storage[i].push_back('\0');
cargs.push_back(argv_storage[i].data());
}
cargs.push_back(nullptr);
pid_t pid = fork();
if (pid == 0) {
execvp(cargs[0], cargs.data());
_exit(127);
}
if (pid < 0) return -1;
int status = 0;
while (waitpid(pid, &status, 0) == -1 && errno == EINTR) {}
return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
}
static std::optional<dsl::IndexDef> load_cached_index(
const std::string& repo_name,
const std::string& channel) {
auto key = sanitize_filename(repo_name + "-" + channel) + ".kap";
auto path = paths::cache_dir() / "indexes" / key;
if (!std::filesystem::exists(path)) return std::nullopt;
std::ifstream in(path);
if (!in) return std::nullopt;
std::ostringstream buf;
buf << in.rdbuf();
try {
return dsl::parse_index(buf.str());
} catch (...) {
return std::nullopt;
}
}
static bool download_index(const dsl::RepoDef& repo,
const std::string& channel,
const std::filesystem::path& cache_path) {
auto url = repo.url;
if (!url.empty() && url.back() != '/') url += '/';
url += channel + "/index.kap";
auto temp = cache_path.string() + ".tmp";
// Try primary URL
int rc = exec_curl_conditional(temp, url, cache_path.string());
if (rc != 0) {
// Try mirrors
for (auto& mirror : repo.mirrors) {
auto murl = mirror;
if (!murl.empty() && murl.back() != '/') murl += '/';
murl += channel + "/index.kap";
rc = exec_curl_conditional(temp, murl, "");
if (rc == 0) break;
}
}
if (rc != 0) return false;
// If curl returned 0 but the file might be empty (304 not modified),
// move temp to cache only if it has content
std::error_code ec;
if (std::filesystem::file_size(temp, ec) > 0 || ec) {
std::filesystem::rename(temp, cache_path, ec);
return !ec;
}
// 304: no change, remove temp
std::filesystem::remove(temp, ec);
return true; // cache is still valid
}
static std::optional<std::string> download_recipe(
const std::string& name,
const dsl::RepoDef& repo,
const std::string& channel) {
auto remote_path = channel + "/" + name + ".kap";
auto temp = paths::temp_dir() / (name + ".kap.tmp");
// Build URL list: primary + mirrors
std::vector<std::string> urls;
auto url = repo.url;
if (!url.empty() && url.back() != '/') url += '/';
urls.push_back(url + remote_path);
for (auto& mirror : repo.mirrors) {
auto murl = mirror;
if (!murl.empty() && murl.back() != '/') murl += '/';
urls.push_back(murl + remote_path);
}
for (auto& u : urls) {
int rc = exec_curl_conditional(temp.string(), u, "");
if (rc == 0) {
std::ifstream in(temp);
if (!in) { std::filesystem::remove(temp); continue; }
std::ostringstream buf;
buf << in.rdbuf();
in.close();
std::filesystem::remove(temp);
return buf.str();
}
}
return std::nullopt;
}
RecipeResult fetch_recipe_from_repos(
const std::string& name,
const std::vector<dsl::RepoDef>& repos) {
RecipeResult result;
// Check cached version first
auto cache_path = paths::packages_dir() / (name + ".kap");
std::string cached_version;
if (std::filesystem::exists(cache_path)) {
std::ifstream in(cache_path);
if (in) {
std::ostringstream buf;
buf << in.rdbuf();
try {
auto pkg = dsl::parse(buf.str());
cached_version = pkg.version;
} catch (...) {}
}
}
// Ensure indexes cache directory exists
std::error_code ec;
std::filesystem::create_directories(paths::cache_dir() / "indexes", ec);
// Sort repos by priority (descending)
std::vector<const dsl::RepoDef*> sorted;
for (auto& r : repos) sorted.push_back(&r);
std::sort(sorted.begin(), sorted.end(),
[](auto* a, auto* b) { return a->priority > b->priority; });
std::string best_content;
std::string best_version;
for (auto* repo : sorted) {
for (auto& channel : repo->channels) {
// Check/update the index for this repo+channel
auto idx_key = sanitize_filename(repo->name + "-" + channel);
auto idx_cache = paths::cache_dir() / "indexes" / (idx_key + ".kap");
// Download fresh index (conditional on cache)
if (!download_index(*repo, channel, idx_cache)) continue;
// Load and search the index
auto idx = load_cached_index(repo->name, channel);
if (!idx) continue;
bool found = false;
for (auto& entry : idx->packages) {
if (entry.name != name) continue;
found = true;
break;
}
if (!found) continue;
// Download the recipe
auto content = download_recipe(name, *repo, channel);
if (!content) continue;
// Parse the downloaded recipe to get version
std::string remote_version;
try {
auto pkg = dsl::parse(*content);
remote_version = pkg.version;
} catch (...) {
continue;
}
if (remote_version > best_version) {
best_version = remote_version;
best_content = std::move(*content);
}
}
}
if (best_content.empty()) {
if (!cached_version.empty()) {
result.ok = true;
result.path = cache_path.string();
result.version = cached_version;
return result;
}
result.error = "package '" + name + "' not found in any repo";
return result;
}
// Update cache if remote is newer
if (best_version > cached_version || cached_version.empty()) {
std::ofstream out(cache_path);
if (!out) {
result.error = "cannot write to cache";
return result;
}
out << best_content;
result.updated = true;
}
result.ok = true;
result.path = cache_path.string();
result.version = best_version;
return result;
}
} // namespace kappa::fetch