From 8ee8ec30dbb4eb42eac7445a8ff3b0cc5e160ff4 Mon Sep 17 00:00:00 2001 From: huntedbytheirs Date: Wed, 19 Aug 2026 18:21:52 -0400 Subject: [PATCH] feat(arch/binary): convert .pkg.tar.zst to Zeta binary package Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- go.mod | 2 + go.sum | 2 + internal/arch/binary/convert.go | 251 ++++++++++++++++++++++++ internal/arch/binary/convert_test.go | 204 +++++++++++++++++++ internal/arch/binary/frontend.go | 7 +- testdata/hello-1.0-1-x86_64.pkg.tar.zst | Bin 286 -> 304 bytes 6 files changed, 462 insertions(+), 4 deletions(-) create mode 100644 go.sum create mode 100644 internal/arch/binary/convert.go create mode 100644 internal/arch/binary/convert_test.go diff --git a/go.mod b/go.mod index 7c7f96f..1a9503b 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module git.spectoria.dev/huntedbytheirs/zeta-reconstruct go 1.26.5 + +require github.com/klauspost/compress v1.19.2 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f7deada --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= diff --git a/internal/arch/binary/convert.go b/internal/arch/binary/convert.go new file mode 100644 index 0000000..9382922 --- /dev/null +++ b/internal/arch/binary/convert.go @@ -0,0 +1,251 @@ +package binary + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/klauspost/compress/zstd" + + "git.spectoria.dev/huntedbytheirs/zeta-reconstruct/internal/depmap" + "git.spectoria.dev/huntedbytheirs/zeta-reconstruct/internal/registry" + "git.spectoria.dev/huntedbytheirs/zeta-reconstruct/internal/tarball" + "git.spectoria.dev/huntedbytheirs/zeta-reconstruct/internal/zeta" +) + +// rootMetadata is the set of Arch metadata members that live at the ROOT of +// a .pkg.tar.zst and are never part of the installed payload. Only these +// names at the root are skipped; a dot-file with the same name deeper in the +// tree is real content. +var rootMetadata = map[string]bool{ + ".PKGINFO": true, + ".BUILDINFO": true, + ".MTREE": true, + ".INSTALL": true, + ".Changelog": true, +} + +// Convert converts an Arch binary package (.pkg.tar.zst) into a Zeta binary +// package: +// +// 1. Open input with a zstd reader and walk the tar members. +// 2. Read and parse the .PKGINFO member (member name ".PKGINFO" or +// "./.PKGINFO"). Its pkgver is the Zeta version, verbatim (epoch+pkgrel +// form, e.g. "1.0-1"). +// 3. Extract every other member (root dot-metadata skipped) into a temp dir, +// preserving paths, modes and symlinks. +// 4. Repack the payload with tarball.Write into +// /packages//-.tar.gz. +// 5. Emit an archive-mode package.lua next to the tarball, with depends +// mapped through depmap.MapDep. +// +// An epoch (":" in pkgver) is warned about and never folded: Zeta versioning +// cannot represent it. +func Convert(input string, opts registry.Options) error { + f, err := os.Open(input) + if err != nil { + return fmt.Errorf("convert: open %s: %w", input, err) + } + zr, err := zstd.NewReader(f) + if err != nil { + _ = f.Close() + return fmt.Errorf("convert: zstd %s: %w", input, err) + } + defer func() { + zr.Close() + _ = f.Close() + }() + tr := tar.NewReader(zr) + + warn := func(msg string) { fmt.Fprintf(os.Stderr, "warning: %s\n", msg) } + + extracted, err := os.MkdirTemp("", "zr-arch-binary-*") + if err != nil { + return fmt.Errorf("convert: temp dir: %w", err) + } + defer os.RemoveAll(extracted) + + var info *PKGInfo + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("convert: read %s: %w", input, err) + } + + name := cleanMemberName(hdr.Name) + + // Root dot-metadata: never extracted. .PKGINFO is read from the + // stream here; the other four are skipped without reading. + if isRootMetadata(name) { + if name == ".PKGINFO" { + content, err := io.ReadAll(tr) + if err != nil { + return fmt.Errorf("convert: read .PKGINFO: %w", err) + } + info, err = Parse(string(content), warn) + if err != nil { + return fmt.Errorf("convert: parse .PKGINFO: %w", err) + } + } + continue + } + + if err := extractMember(extracted, name, hdr, tr); err != nil { + return err + } + } + + if info == nil { + return errors.New("convert: archive has no .PKGINFO member") + } + if info.Name == "" { + return errors.New("convert: .PKGINFO has no pkgname") + } + + if strings.Contains(info.Version, ":") { + warn(fmt.Sprintf("package %s has epoch in version; epoch is not representable in Zeta versioning", info.Name)) + } + + // Deps: map each declared dependency through the Arch→Zeta name map. + // Dropped deps are skipped (MapDep already warned). + var deps []string + for _, d := range info.Depends { + if mapped, ok := depmap.MapDep(d, warn); ok { + deps = append(deps, mapped) + } + } + + pkgDir := filepath.Join(opts.Output, "packages", info.Name) + if err := os.MkdirAll(pkgDir, 0o755); err != nil { + return fmt.Errorf("convert: mkdir %s: %w", pkgDir, err) + } + + tarballPath := filepath.Join(pkgDir, info.Name+"-"+info.Version+".tar.gz") + sum, err := tarball.Write(extracted, tarballPath) + if err != nil { + return fmt.Errorf("convert: repack payload: %w", err) + } + + m := &zeta.Manifest{ + Name: info.Name, + Version: info.Version, + Summary: info.Desc, + URL: opts.Repo + "/packages/" + info.Name + "/" + info.Name + "-" + info.Version + ".tar.gz", + SHA256: sum, + Deps: deps, + Archive: &zeta.Archive{Strip: 1}, + } + + src, err := m.Emit() + if err != nil { + return fmt.Errorf("convert: emit manifest: %w", err) + } + manifestPath := filepath.Join(pkgDir, "package.lua") + if err := os.WriteFile(manifestPath, []byte(src), 0o644); err != nil { + return fmt.Errorf("convert: write %s: %w", manifestPath, err) + } + return nil +} + +// cleanMemberName strips the "./" prefix (and any leading slashes) GNU tar +// prepends, yielding the canonical root-relative member name. +func cleanMemberName(name string) string { + for strings.HasPrefix(name, "./") { + name = name[2:] + } + return name +} + +// isRootMetadata reports whether name is one of the Arch metadata members at +// the ROOT of the archive (no directory component). +func isRootMetadata(name string) bool { + if name == "" || name == "." { + return false + } + if strings.Contains(name, "/") { + return false + } + return rootMetadata[name] +} + +// extractMember extracts a single tar member under root, preserving paths, +// modes and symlinks. Directories are created; hard links, FIFOs and device +// nodes are skipped, matching tarball.Write's policy. +func extractMember(root, name string, hdr *tar.Header, r io.Reader) error { + if name == "" || name == "." { + return nil // the archive root entry itself + } + if strings.HasPrefix(name, "/") || !relativeInside(name) { + return fmt.Errorf("convert: member %q escapes the extraction root", name) + } + + target := filepath.Join(root, filepath.FromSlash(name)) + switch hdr.Typeflag { + case tar.TypeDir: + mode := os.FileMode(hdr.Mode).Perm() + if mode == 0 { + mode = 0o755 + } + return os.MkdirAll(target, mode) + + case tar.TypeReg, tar.TypeRegA: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + mode := os.FileMode(hdr.Mode).Perm() + if mode == 0 { + mode = 0o644 + } + _ = os.Remove(target) // never write through a pre-existing symlink + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("convert: extract %s: %w", name, err) + } + if _, err := io.Copy(f, r); err != nil { + _ = f.Close() + return fmt.Errorf("convert: extract %s: %w", name, err) + } + return f.Close() + + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + _ = os.Remove(target) + if err := os.Symlink(hdr.Linkname, target); err != nil { + return fmt.Errorf("convert: extract symlink %s: %w", name, err) + } + return nil + + default: + // Hard links, FIFOs, devices, sparse files: skipped like the + // repack step. + return nil + } +} + +// relativeInside reports whether the slash-separated relative path p never +// climbs above its root. Port of ZETA/lib/path.lua relative_inside. +func relativeInside(p string) bool { + depth := 0 + for _, seg := range strings.Split(p, "/") { + switch seg { + case "..": + depth-- + if depth < 0 { + return false + } + case "", ".": + default: + depth++ + } + } + return true +} diff --git a/internal/arch/binary/convert_test.go b/internal/arch/binary/convert_test.go new file mode 100644 index 0000000..5811c1d --- /dev/null +++ b/internal/arch/binary/convert_test.go @@ -0,0 +1,204 @@ +package binary + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + + "git.spectoria.dev/huntedbytheirs/zeta-reconstruct/internal/registry" +) + +const fixturePkg = "../../../testdata/hello-1.0-1-x86_64.pkg.tar.zst" + +func TestConvert(t *testing.T) { + out := t.TempDir() + err := Convert(fixturePkg, registry.Options{Output: out, Repo: "https://example.org/repo"}) + if err != nil { + t.Fatalf("Convert() error = %v, want nil", err) + } + + manifestPath := filepath.Join(out, "packages", "hello", "package.lua") + tarballPath := filepath.Join(out, "packages", "hello", "hello-1.0-1.tar.gz") + + content, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("package.lua not written: %v", err) + } + src := string(content) + + if !strings.Contains(src, "archive = { strip = 1 }") { + t.Errorf("package.lua does not contain archive mode marker:\n%s", src) + } + if !strings.Contains(src, `name = "hello"`) { + t.Errorf("package.lua does not contain name field:\n%s", src) + } + + if _, err := os.Stat(tarballPath); err != nil { + t.Fatalf("tarball not written: %v", err) + } + + // The sha256 field must equal a fresh hash of the written tarball. + got := extractField(t, src, "sha256") + want := fileSHA256(t, tarballPath) + if got != want { + t.Errorf("package.lua sha256 = %q, sha256(tarball) = %q", got, want) + } + + // Deps: glibc dropped, zlib renamed to libz. + if !strings.Contains(src, `"libz"`) { + t.Errorf("package.lua deps does not contain mapped libz:\n%s", src) + } + if strings.Contains(src, "glibc") { + t.Errorf("package.lua still contains dropped glibc:\n%s", src) + } + + // Cross-tool validation: the emitted manifest must load through the + // real Zeta manifest loader. + lua, err := exec.LookPath("lua") + if err != nil { + t.Skip("lua not available; skipping cross-tool validation") + } + probe := `package.path="/home/specter/Public/ZETA/lib/?.lua;"..package.path; ` + + `local m=require("manifest"); assert(m.load(` + quoteLua(manifestPath) + `))` + cmd := exec.Command(lua, "-e", probe) + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("real Zeta manifest.load rejected the emitted manifest: %v\n%s", err, b) + } + + // The tarball must be a valid gzip+tar whose content matches the input + // payload (usr/bin/hello). + verifyTarballPayload(t, tarballPath) +} + +func TestConvertMissingPKGINFO(t *testing.T) { + // A zstd-compressed tar without a .PKGINFO member. + var buf strings.Builder + tw := tar.NewWriter(&buf) + body := "#!/bin/sh\n" + if err := tw.WriteHeader(&tar.Header{Name: "usr/bin/x", Mode: 0o755, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + enc, err := zstd.NewWriter(nil) + if err != nil { + t.Fatal(err) + } + compressed := enc.EncodeAll([]byte(buf.String()), nil) + + pkg := filepath.Join(t.TempDir(), "noinfo.pkg.tar.zst") + if err := os.WriteFile(pkg, compressed, 0o644); err != nil { + t.Fatal(err) + } + + err = Convert(pkg, registry.Options{Output: t.TempDir(), Repo: "https://example.org/repo"}) + if err == nil { + t.Fatal("Convert() error = nil, want error for archive without .PKGINFO") + } + if !strings.Contains(err.Error(), "PKGINFO") { + t.Errorf("Convert() error = %q, want it to name the missing .PKGINFO", err) + } +} + +func TestConvertMissingInput(t *testing.T) { + err := Convert(filepath.Join(t.TempDir(), "nope.pkg.tar.zst"), + registry.Options{Output: t.TempDir(), Repo: "https://example.org/repo"}) + if err == nil { + t.Fatal("Convert() error = nil, want error for missing input file") + } +} + +// extractField pulls the value of a `key = "value",` line from an emitted +// package.lua. +func extractField(t *testing.T, src, key string) string { + t.Helper() + for _, line := range strings.Split(src, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, key) { + continue + } + i := strings.Index(line, `"`) + j := strings.LastIndex(line, `"`) + if i < 0 || j <= i { + t.Fatalf("field %q has no quoted value in %q", key, line) + } + return line[i+1 : j] + } + t.Fatalf("field %q not found in package.lua", key) + return "" +} + +// fileSHA256 returns the lowercase hex sha256 of the file at path. +func fileSHA256(t *testing.T, path string) string { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + t.Fatal(err) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// quoteLua renders a path as a safe single-quoted Lua string literal. +func quoteLua(s string) string { + return `'` + strings.ReplaceAll(s, `\`, `\\`) + `'` +} + +// verifyTarballPayload re-opens the emitted tarball and asserts the payload +// file survived with the right content. +func verifyTarballPayload(t *testing.T, path string) { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + + var found bool + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if strings.TrimPrefix(hdr.Name, "./") == "usr/bin/hello" { + found = true + b, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), "hello") { + t.Errorf("payload usr/bin/hello content = %q", b) + } + } + } + if !found { + t.Error("payload member usr/bin/hello missing from tarball") + } +} diff --git a/internal/arch/binary/frontend.go b/internal/arch/binary/frontend.go index 2342770..01fe6e5 100644 --- a/internal/arch/binary/frontend.go +++ b/internal/arch/binary/frontend.go @@ -3,8 +3,6 @@ package binary import ( - "errors" - "git.spectoria.dev/huntedbytheirs/zeta-reconstruct/internal/registry" ) @@ -13,9 +11,10 @@ type frontend struct{} func (frontend) Name() string { return "arch-binary" } +// Convert converts an Arch .pkg.tar.zst into a Zeta binary package +// (payload tarball + archive-mode package.lua) under opts.Output. func (frontend) Convert(input string, opts registry.Options) error { - // Implemented in a later todo (.pkg.tar.zst -> tarball + package.lua). - return errors.New("not implemented") + return Convert(input, opts) } func init() { diff --git a/testdata/hello-1.0-1-x86_64.pkg.tar.zst b/testdata/hello-1.0-1-x86_64.pkg.tar.zst index 6fa0a867b341abf318d30aff0f01ca765d6f6bbb..6f82d21a3797b7c90b09e6ec914d9e90f946b8e3 100644 GIT binary patch literal 304 zcmV-00nh#@wJ-eySRDxfvJWvIP)qZ}oq0Abrpw81N>wLRaKl!qFNb#0KBofJ81@}g zK^`dJ59VhbXCx{sh#=rwRR*?D-tTTP2 z%zJpH2rRuassnt0jN z!GLyxbOMTc02KiEk(3wV0|s>uh#s=R0(`-K&X~z*f{GCbf-8`kzrb?UC*uYL+`ztW CB#giS literal 286 zcmV+(0pb2AwJ-eySd9n(B91H{P)qXwy2+c&#&J0lrdM@B1vhM!JT2~|eNF|cG3+}M zIUFGrAb`6Kw6z>tib)>S(`L2tv{4B$=imqy`xZGKqWChoG8edTr3&b*71_7SH)w$$ z<z1X6D0i zu#kYD(2!I*@AdFW&+fxYo746*cQ+r_-OZY<)`y$X%zRoin3<1C0U;m|1Cc-^4E3D* z>>&vO1jHEtD&VmJRLKq}QVjS41|UG0ReAtSuqR$Nbuge$zz3SNQ7}``Mc{&IsR64( kD;}VGfCdZj1^YQ;C#L}_1{?^kKx+O1%T=F>8`y%{a6-9$D*ylh