- fetch_recipe_from_repos() iterates repos by priority, channels, with
mirror fallback — caches indexes at /cache/indexes/
- Conditional HTTP downloads (curl -z) to only re-fetch stale indexes
- build_registry and fetch-package prefer repos { } over remotes = [...]
when both are present
- Fallback chain: local .kap → repos (index-aware) → remotes (legacy)
361 lines
11 KiB
C++
361 lines
11 KiB
C++
#include "kappa/fetch/recipe.hpp"
|
|
|
|
#include "kappa/dsl/parser.hpp"
|
|
#include "kappa/dsl/system.hpp"
|
|
#include "kappa/paths.hpp"
|
|
|
|
#include <sys/wait.h>
|
|
#include <unistd.h>
|
|
#include <filesystem>
|
|
#include <format>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <sstream>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
namespace kappa::fetch {
|
|
|
|
namespace {
|
|
int exec_curl(const std::string& output_path, const std::string& url) {
|
|
pid_t pid = fork();
|
|
if (pid == 0) {
|
|
execlp("curl", "curl", "-Lsf", "-o", output_path.c_str(), url.c_str(), nullptr);
|
|
_exit(127);
|
|
}
|
|
if (pid < 0) return -1;
|
|
int status = 0;
|
|
while (waitpid(pid, &status, 0) == -1 && errno == EINTR) {}
|
|
return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
|
|
}
|
|
} // namespace
|
|
|
|
RecipeResult fetch_recipe(const std::string& name,
|
|
const std::vector<std::string>& remotes) {
|
|
RecipeResult result;
|
|
|
|
// Check cached version first
|
|
auto cache_path = paths::packages_dir() / (name + ".kap");
|
|
std::string cached_version;
|
|
if (std::filesystem::exists(cache_path)) {
|
|
std::ifstream in(cache_path);
|
|
if (in) {
|
|
std::ostringstream buf;
|
|
buf << in.rdbuf();
|
|
try {
|
|
auto pkg = dsl::parse(buf.str());
|
|
cached_version = pkg.version;
|
|
} catch (...) {
|
|
// Corrupt cache — will re-download
|
|
}
|
|
}
|
|
}
|
|
|
|
// Try each remote
|
|
std::string best_content;
|
|
std::string best_version;
|
|
std::string best_url;
|
|
|
|
for (auto& remote : remotes) {
|
|
auto url = remote;
|
|
if (!url.empty() && url.back() != '/') url += '/';
|
|
url += name + ".kap";
|
|
|
|
// Download to temp
|
|
auto temp_path = paths::temp_dir() / (name + ".kap.tmp");
|
|
int rc = exec_curl(temp_path.string(), url);
|
|
if (rc != 0) continue;
|
|
|
|
// Parse downloaded file
|
|
std::ifstream in(temp_path);
|
|
if (!in) { std::filesystem::remove(temp_path); continue; }
|
|
std::ostringstream buf;
|
|
buf << in.rdbuf();
|
|
in.close();
|
|
|
|
std::string remote_version;
|
|
try {
|
|
auto pkg = dsl::parse(buf.str());
|
|
remote_version = pkg.version;
|
|
} catch (...) {
|
|
std::filesystem::remove(temp_path);
|
|
continue;
|
|
}
|
|
|
|
// Compare versions — keep the best (highest)
|
|
// Simple string comparison for now; semver later
|
|
if (remote_version > best_version) {
|
|
best_version = remote_version;
|
|
best_content = buf.str();
|
|
best_url = url;
|
|
}
|
|
|
|
std::filesystem::remove(temp_path);
|
|
}
|
|
|
|
if (best_content.empty()) {
|
|
if (!cached_version.empty()) {
|
|
// No remote available but have cache
|
|
result.ok = true;
|
|
result.path = cache_path.string();
|
|
result.version = cached_version;
|
|
result.updated = false;
|
|
return result;
|
|
}
|
|
result.error = "package '" + name + "' not found in any remote";
|
|
return result;
|
|
}
|
|
|
|
// Update cache if remote is newer
|
|
if (best_version > cached_version || cached_version.empty()) {
|
|
std::error_code ec;
|
|
std::filesystem::create_directories(paths::packages_dir(), ec);
|
|
std::ofstream out(cache_path);
|
|
if (!out) {
|
|
result.error = "cannot write to cache";
|
|
return result;
|
|
}
|
|
out << best_content;
|
|
result.updated = true;
|
|
}
|
|
|
|
result.ok = true;
|
|
result.path = cache_path.string();
|
|
result.version = best_version;
|
|
return result;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Index-aware recipe fetching
|
|
// ---------------------------------------------------------------------------
|
|
|
|
static std::string sanitize_filename(const std::string& s) {
|
|
std::string out = s;
|
|
for (auto& c : out) {
|
|
if (c == '/' || c == '\\' || c == ':') c = '_';
|
|
}
|
|
return out;
|
|
}
|
|
|
|
static int exec_curl_conditional(const std::string& output_path,
|
|
const std::string& url,
|
|
const std::string& time_cond_path) {
|
|
std::vector<std::string> argv = {"curl", "-Lsf", "-o", output_path};
|
|
if (!time_cond_path.empty()) {
|
|
argv.push_back("-z");
|
|
argv.push_back(time_cond_path);
|
|
}
|
|
argv.push_back(url);
|
|
|
|
std::vector<std::vector<char>> argv_storage(argv.size());
|
|
std::vector<char*> cargs;
|
|
for (size_t i = 0; i < argv.size(); ++i) {
|
|
argv_storage[i].assign(argv[i].begin(), argv[i].end());
|
|
argv_storage[i].push_back('\0');
|
|
cargs.push_back(argv_storage[i].data());
|
|
}
|
|
cargs.push_back(nullptr);
|
|
|
|
pid_t pid = fork();
|
|
if (pid == 0) {
|
|
execvp(cargs[0], cargs.data());
|
|
_exit(127);
|
|
}
|
|
if (pid < 0) return -1;
|
|
int status = 0;
|
|
while (waitpid(pid, &status, 0) == -1 && errno == EINTR) {}
|
|
return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
|
|
}
|
|
|
|
static std::optional<dsl::IndexDef> load_cached_index(
|
|
const std::string& repo_name,
|
|
const std::string& channel) {
|
|
auto key = sanitize_filename(repo_name + "-" + channel) + ".kap";
|
|
auto path = paths::cache_dir() / "indexes" / key;
|
|
if (!std::filesystem::exists(path)) return std::nullopt;
|
|
|
|
std::ifstream in(path);
|
|
if (!in) return std::nullopt;
|
|
std::ostringstream buf;
|
|
buf << in.rdbuf();
|
|
try {
|
|
return dsl::parse_index(buf.str());
|
|
} catch (...) {
|
|
return std::nullopt;
|
|
}
|
|
}
|
|
|
|
static bool download_index(const dsl::RepoDef& repo,
|
|
const std::string& channel,
|
|
const std::filesystem::path& cache_path) {
|
|
auto url = repo.url;
|
|
if (!url.empty() && url.back() != '/') url += '/';
|
|
url += channel + "/index.kap";
|
|
|
|
auto temp = cache_path.string() + ".tmp";
|
|
|
|
// Try primary URL
|
|
int rc = exec_curl_conditional(temp, url, cache_path.string());
|
|
if (rc != 0) {
|
|
// Try mirrors
|
|
for (auto& mirror : repo.mirrors) {
|
|
auto murl = mirror;
|
|
if (!murl.empty() && murl.back() != '/') murl += '/';
|
|
murl += channel + "/index.kap";
|
|
rc = exec_curl_conditional(temp, murl, "");
|
|
if (rc == 0) break;
|
|
}
|
|
}
|
|
|
|
if (rc != 0) return false;
|
|
|
|
// If curl returned 0 but the file might be empty (304 not modified),
|
|
// move temp to cache only if it has content
|
|
std::error_code ec;
|
|
if (std::filesystem::file_size(temp, ec) > 0 || ec) {
|
|
std::filesystem::rename(temp, cache_path, ec);
|
|
return !ec;
|
|
}
|
|
|
|
// 304: no change, remove temp
|
|
std::filesystem::remove(temp, ec);
|
|
return true; // cache is still valid
|
|
}
|
|
|
|
static std::optional<std::string> download_recipe(
|
|
const std::string& name,
|
|
const dsl::RepoDef& repo,
|
|
const std::string& channel) {
|
|
auto remote_path = channel + "/" + name + ".kap";
|
|
auto temp = paths::temp_dir() / (name + ".kap.tmp");
|
|
|
|
// Build URL list: primary + mirrors
|
|
std::vector<std::string> urls;
|
|
auto url = repo.url;
|
|
if (!url.empty() && url.back() != '/') url += '/';
|
|
urls.push_back(url + remote_path);
|
|
for (auto& mirror : repo.mirrors) {
|
|
auto murl = mirror;
|
|
if (!murl.empty() && murl.back() != '/') murl += '/';
|
|
urls.push_back(murl + remote_path);
|
|
}
|
|
|
|
for (auto& u : urls) {
|
|
int rc = exec_curl_conditional(temp.string(), u, "");
|
|
if (rc == 0) {
|
|
std::ifstream in(temp);
|
|
if (!in) { std::filesystem::remove(temp); continue; }
|
|
std::ostringstream buf;
|
|
buf << in.rdbuf();
|
|
in.close();
|
|
std::filesystem::remove(temp);
|
|
return buf.str();
|
|
}
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
RecipeResult fetch_recipe_from_repos(
|
|
const std::string& name,
|
|
const std::vector<dsl::RepoDef>& repos) {
|
|
RecipeResult result;
|
|
|
|
// Check cached version first
|
|
auto cache_path = paths::packages_dir() / (name + ".kap");
|
|
std::string cached_version;
|
|
if (std::filesystem::exists(cache_path)) {
|
|
std::ifstream in(cache_path);
|
|
if (in) {
|
|
std::ostringstream buf;
|
|
buf << in.rdbuf();
|
|
try {
|
|
auto pkg = dsl::parse(buf.str());
|
|
cached_version = pkg.version;
|
|
} catch (...) {}
|
|
}
|
|
}
|
|
|
|
// Ensure indexes cache directory exists
|
|
std::error_code ec;
|
|
std::filesystem::create_directories(paths::cache_dir() / "indexes", ec);
|
|
|
|
// Sort repos by priority (descending)
|
|
std::vector<const dsl::RepoDef*> sorted;
|
|
for (auto& r : repos) sorted.push_back(&r);
|
|
std::sort(sorted.begin(), sorted.end(),
|
|
[](auto* a, auto* b) { return a->priority > b->priority; });
|
|
|
|
std::string best_content;
|
|
std::string best_version;
|
|
|
|
for (auto* repo : sorted) {
|
|
for (auto& channel : repo->channels) {
|
|
// Check/update the index for this repo+channel
|
|
auto idx_key = sanitize_filename(repo->name + "-" + channel);
|
|
auto idx_cache = paths::cache_dir() / "indexes" / (idx_key + ".kap");
|
|
|
|
// Download fresh index (conditional on cache)
|
|
if (!download_index(*repo, channel, idx_cache)) continue;
|
|
|
|
// Load and search the index
|
|
auto idx = load_cached_index(repo->name, channel);
|
|
if (!idx) continue;
|
|
|
|
bool found = false;
|
|
for (auto& entry : idx->packages) {
|
|
if (entry.name != name) continue;
|
|
found = true;
|
|
break;
|
|
}
|
|
if (!found) continue;
|
|
|
|
// Download the recipe
|
|
auto content = download_recipe(name, *repo, channel);
|
|
if (!content) continue;
|
|
|
|
// Parse the downloaded recipe to get version
|
|
std::string remote_version;
|
|
try {
|
|
auto pkg = dsl::parse(*content);
|
|
remote_version = pkg.version;
|
|
} catch (...) {
|
|
continue;
|
|
}
|
|
|
|
if (remote_version > best_version) {
|
|
best_version = remote_version;
|
|
best_content = std::move(*content);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (best_content.empty()) {
|
|
if (!cached_version.empty()) {
|
|
result.ok = true;
|
|
result.path = cache_path.string();
|
|
result.version = cached_version;
|
|
return result;
|
|
}
|
|
result.error = "package '" + name + "' not found in any repo";
|
|
return result;
|
|
}
|
|
|
|
// Update cache if remote is newer
|
|
if (best_version > cached_version || cached_version.empty()) {
|
|
std::ofstream out(cache_path);
|
|
if (!out) {
|
|
result.error = "cannot write to cache";
|
|
return result;
|
|
}
|
|
out << best_content;
|
|
result.updated = true;
|
|
}
|
|
|
|
result.ok = true;
|
|
result.path = cache_path.string();
|
|
result.version = best_version;
|
|
return result;
|
|
}
|
|
|
|
} // namespace kappa::fetch
|