feat: add index-aware recipe resolution with repo support
- fetch_recipe_from_repos() iterates repos by priority, channels, with
mirror fallback — caches indexes at /cache/indexes/
- Conditional HTTP downloads (curl -z) to only re-fetch stale indexes
- build_registry and fetch-package prefer repos { } over remotes = [...]
when both are present
- Fallback chain: local .kap → repos (index-aware) → remotes (legacy)
This commit is contained in:
@@ -125,4 +125,236 @@ RecipeResult fetch_recipe(const std::string& name,
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Index-aware recipe fetching
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static std::string sanitize_filename(const std::string& s) {
|
||||
std::string out = s;
|
||||
for (auto& c : out) {
|
||||
if (c == '/' || c == '\\' || c == ':') c = '_';
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static int exec_curl_conditional(const std::string& output_path,
|
||||
const std::string& url,
|
||||
const std::string& time_cond_path) {
|
||||
std::vector<std::string> argv = {"curl", "-Lsf", "-o", output_path};
|
||||
if (!time_cond_path.empty()) {
|
||||
argv.push_back("-z");
|
||||
argv.push_back(time_cond_path);
|
||||
}
|
||||
argv.push_back(url);
|
||||
|
||||
std::vector<std::vector<char>> argv_storage(argv.size());
|
||||
std::vector<char*> cargs;
|
||||
for (size_t i = 0; i < argv.size(); ++i) {
|
||||
argv_storage[i].assign(argv[i].begin(), argv[i].end());
|
||||
argv_storage[i].push_back('\0');
|
||||
cargs.push_back(argv_storage[i].data());
|
||||
}
|
||||
cargs.push_back(nullptr);
|
||||
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
execvp(cargs[0], cargs.data());
|
||||
_exit(127);
|
||||
}
|
||||
if (pid < 0) return -1;
|
||||
int status = 0;
|
||||
while (waitpid(pid, &status, 0) == -1 && errno == EINTR) {}
|
||||
return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
|
||||
}
|
||||
|
||||
static std::optional<dsl::IndexDef> load_cached_index(
|
||||
const std::string& repo_name,
|
||||
const std::string& channel) {
|
||||
auto key = sanitize_filename(repo_name + "-" + channel) + ".kap";
|
||||
auto path = paths::cache_dir() / "indexes" / key;
|
||||
if (!std::filesystem::exists(path)) return std::nullopt;
|
||||
|
||||
std::ifstream in(path);
|
||||
if (!in) return std::nullopt;
|
||||
std::ostringstream buf;
|
||||
buf << in.rdbuf();
|
||||
try {
|
||||
return dsl::parse_index(buf.str());
|
||||
} catch (...) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
static bool download_index(const dsl::RepoDef& repo,
|
||||
const std::string& channel,
|
||||
const std::filesystem::path& cache_path) {
|
||||
auto url = repo.url;
|
||||
if (!url.empty() && url.back() != '/') url += '/';
|
||||
url += channel + "/index.kap";
|
||||
|
||||
auto temp = cache_path.string() + ".tmp";
|
||||
|
||||
// Try primary URL
|
||||
int rc = exec_curl_conditional(temp, url, cache_path.string());
|
||||
if (rc != 0) {
|
||||
// Try mirrors
|
||||
for (auto& mirror : repo.mirrors) {
|
||||
auto murl = mirror;
|
||||
if (!murl.empty() && murl.back() != '/') murl += '/';
|
||||
murl += channel + "/index.kap";
|
||||
rc = exec_curl_conditional(temp, murl, "");
|
||||
if (rc == 0) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (rc != 0) return false;
|
||||
|
||||
// If curl returned 0 but the file might be empty (304 not modified),
|
||||
// move temp to cache only if it has content
|
||||
std::error_code ec;
|
||||
if (std::filesystem::file_size(temp, ec) > 0 || ec) {
|
||||
std::filesystem::rename(temp, cache_path, ec);
|
||||
return !ec;
|
||||
}
|
||||
|
||||
// 304: no change, remove temp
|
||||
std::filesystem::remove(temp, ec);
|
||||
return true; // cache is still valid
|
||||
}
|
||||
|
||||
static std::optional<std::string> download_recipe(
|
||||
const std::string& name,
|
||||
const dsl::RepoDef& repo,
|
||||
const std::string& channel) {
|
||||
auto remote_path = channel + "/" + name + ".kap";
|
||||
auto temp = paths::temp_dir() / (name + ".kap.tmp");
|
||||
|
||||
// Build URL list: primary + mirrors
|
||||
std::vector<std::string> urls;
|
||||
auto url = repo.url;
|
||||
if (!url.empty() && url.back() != '/') url += '/';
|
||||
urls.push_back(url + remote_path);
|
||||
for (auto& mirror : repo.mirrors) {
|
||||
auto murl = mirror;
|
||||
if (!murl.empty() && murl.back() != '/') murl += '/';
|
||||
urls.push_back(murl + remote_path);
|
||||
}
|
||||
|
||||
for (auto& u : urls) {
|
||||
int rc = exec_curl_conditional(temp.string(), u, "");
|
||||
if (rc == 0) {
|
||||
std::ifstream in(temp);
|
||||
if (!in) { std::filesystem::remove(temp); continue; }
|
||||
std::ostringstream buf;
|
||||
buf << in.rdbuf();
|
||||
in.close();
|
||||
std::filesystem::remove(temp);
|
||||
return buf.str();
|
||||
}
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
RecipeResult fetch_recipe_from_repos(
|
||||
const std::string& name,
|
||||
const std::vector<dsl::RepoDef>& repos) {
|
||||
RecipeResult result;
|
||||
|
||||
// Check cached version first
|
||||
auto cache_path = paths::packages_dir() / (name + ".kap");
|
||||
std::string cached_version;
|
||||
if (std::filesystem::exists(cache_path)) {
|
||||
std::ifstream in(cache_path);
|
||||
if (in) {
|
||||
std::ostringstream buf;
|
||||
buf << in.rdbuf();
|
||||
try {
|
||||
auto pkg = dsl::parse(buf.str());
|
||||
cached_version = pkg.version;
|
||||
} catch (...) {}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure indexes cache directory exists
|
||||
std::error_code ec;
|
||||
std::filesystem::create_directories(paths::cache_dir() / "indexes", ec);
|
||||
|
||||
// Sort repos by priority (descending)
|
||||
std::vector<const dsl::RepoDef*> sorted;
|
||||
for (auto& r : repos) sorted.push_back(&r);
|
||||
std::sort(sorted.begin(), sorted.end(),
|
||||
[](auto* a, auto* b) { return a->priority > b->priority; });
|
||||
|
||||
std::string best_content;
|
||||
std::string best_version;
|
||||
|
||||
for (auto* repo : sorted) {
|
||||
for (auto& channel : repo->channels) {
|
||||
// Check/update the index for this repo+channel
|
||||
auto idx_key = sanitize_filename(repo->name + "-" + channel);
|
||||
auto idx_cache = paths::cache_dir() / "indexes" / (idx_key + ".kap");
|
||||
|
||||
// Download fresh index (conditional on cache)
|
||||
if (!download_index(*repo, channel, idx_cache)) continue;
|
||||
|
||||
// Load and search the index
|
||||
auto idx = load_cached_index(repo->name, channel);
|
||||
if (!idx) continue;
|
||||
|
||||
bool found = false;
|
||||
for (auto& entry : idx->packages) {
|
||||
if (entry.name != name) continue;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
if (!found) continue;
|
||||
|
||||
// Download the recipe
|
||||
auto content = download_recipe(name, *repo, channel);
|
||||
if (!content) continue;
|
||||
|
||||
// Parse the downloaded recipe to get version
|
||||
std::string remote_version;
|
||||
try {
|
||||
auto pkg = dsl::parse(*content);
|
||||
remote_version = pkg.version;
|
||||
} catch (...) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (remote_version > best_version) {
|
||||
best_version = remote_version;
|
||||
best_content = std::move(*content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (best_content.empty()) {
|
||||
if (!cached_version.empty()) {
|
||||
result.ok = true;
|
||||
result.path = cache_path.string();
|
||||
result.version = cached_version;
|
||||
return result;
|
||||
}
|
||||
result.error = "package '" + name + "' not found in any repo";
|
||||
return result;
|
||||
}
|
||||
|
||||
// Update cache if remote is newer
|
||||
if (best_version > cached_version || cached_version.empty()) {
|
||||
std::ofstream out(cache_path);
|
||||
if (!out) {
|
||||
result.error = "cannot write to cache";
|
||||
return result;
|
||||
}
|
||||
out << best_content;
|
||||
result.updated = true;
|
||||
}
|
||||
|
||||
result.ok = true;
|
||||
result.path = cache_path.string();
|
||||
result.version = best_version;
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace kappa::fetch
|
||||
|
||||
Reference in New Issue
Block a user