feat: support extraction of multiple package types

internal/archive/archive_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/canonical/chisel/internal/archive"
 	"github.com/canonical/chisel/internal/archive/testarchive"
+	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/tarball"
 	"github.com/canonical/chisel/internal/testutil"
 )
@@ -1235,7 +1236,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) {
 	c.Assert(info.Name, DeepEquals, test.pkg)
 	c.Assert(info.Arch, DeepEquals, arch)
 
-	err = tarball.Extract(pkg, &tarball.ExtractOptions{
+	err = tarball.Extract(deb.OpenTar(pkg), &tarball.ExtractOptions{
 		Package:   test.pkg,
 		TargetDir: extractDir,
 		Extract: map[string][]tarball.ExtractInfo{

internal/bin/extract.go

@@ -0,0 +1,27 @@
+package bin
+
+import (
+	"io"
+
+	"github.com/ulikunitz/xz"
+)
+
+// DataReader takes a Reader for a bin package (an XZ-compressed tarball) and
+// returns a Reader to the inner tarball.
+func DataReader(pkgReader io.Reader) (io.ReadCloser, error) {
+	xzReader, err := xz.NewReader(pkgReader)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(xzReader), nil
+}
+
+func OpenTar(pkgReader io.ReadSeeker) func() (io.ReadCloser, error) {
+	return func() (io.ReadCloser, error) {
+		_, err := pkgReader.Seek(0, io.SeekStart)
+		if err != nil {
+			return nil, err
+		}
+		return DataReader(pkgReader)
+	}
+}

internal/bin/extract_test.go

@@ -0,0 +1,41 @@
+package bin_test
+
+import (
+	"archive/tar"
+	"bytes"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/internal/bin"
+	"github.com/canonical/chisel/internal/testutil"
+)
+
+func (s *S) TestDataReader(c *C) {
+	data := testutil.MustMakeBin([]testutil.TarEntry{
+		testutil.Dir(0o755, "./"),
+		testutil.Reg(0o644, "./file", "content"),
+	})
+
+	dataReader, err := bin.DataReader(bytes.NewReader(data))
+	c.Assert(err, IsNil)
+	tarReader := tar.NewReader(dataReader)
+	_, err = tarReader.Next()
+	c.Assert(err, IsNil)
+}
+
+func (s *S) TestDataReaderInvalid(c *C) {
+	_, err := bin.DataReader(bytes.NewReader([]byte("not an xz stream")))
+	c.Assert(err, ErrorMatches, "xz.*")
+}
+
+func (s *S) TestOpenTar(c *C) {
+	data := testutil.MustMakeBin([]testutil.TarEntry{
+		testutil.Dir(0o755, "./"),
+		testutil.Reg(0o644, "./file", "content"),
+	})
+	pkgReader := testutil.ReadSeekNopCloser(bytes.NewReader(data))
+	openTar := bin.OpenTar(pkgReader)
+
+	// Each call must return a fresh stream over the same content.
+	testutil.CheckOpenTar(c, openTar)
+}

internal/bin/log.go

@@ -0,0 +1,53 @@
+package bin
+
+import (
+	"fmt"
+	"sync"
+)
+
+// Avoid importing the log type information unnecessarily.  There's a small cost
+// associated with using an interface rather than the type.  Depending on how
+// often the logger is plugged in, it would be worth using the type instead.
+type log_Logger interface {
+	Output(calldepth int, s string) error
+}
+
+var globalLoggerLock sync.Mutex
+var globalLogger log_Logger
+var globalDebug bool
+
+// Specify the *log.Logger object where log messages should be sent to.
+func SetLogger(logger log_Logger) {
+	globalLoggerLock.Lock()
+	globalLogger = logger
+	globalLoggerLock.Unlock()
+}
+
+// Enable the delivery of debug messages to the logger.  Only meaningful
+// if a logger is also set.
+func SetDebug(debug bool) {
+	globalLoggerLock.Lock()
+	globalDebug = debug
+	globalLoggerLock.Unlock()
+}
+
+// logf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf.
+func logf(format string, args ...any) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}
+
+// debugf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf, but only if debugging was
+// enabled via SetDebug.
+func debugf(format string, args ...any) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalDebug && globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}

internal/bin/suite_test.go

@@ -0,0 +1,13 @@
+package bin_test
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type S struct{}
+
+var _ = Suite(&S{})

internal/deb/extract.go

@@ -47,3 +47,13 @@ func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) {
 
 	return dataReader, nil
 }
+
+func OpenTar(pkgReader io.ReadSeeker) func() (io.ReadCloser, error) {
+	return func() (io.ReadCloser, error) {
+		_, err := pkgReader.Seek(0, io.SeekStart)
+		if err != nil {
+			return nil, err
+		}
+		return DataReader(pkgReader)
+	}
+}

internal/deb/extract_test.go

@@ -0,0 +1,18 @@
+package deb_test
+
+import (
+	"bytes"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/internal/deb"
+	"github.com/canonical/chisel/internal/testutil"
+)
+
+func (s *S) TestOpenTar(c *C) {
+	pkgReader := testutil.ReadSeekNopCloser(bytes.NewReader(testutil.PackageData["test-package"]))
+	openTar := deb.OpenTar(pkgReader)
+
+	// Each call must return a fresh stream over the same content.
+	testutil.CheckOpenTar(c, openTar)
+}

internal/slicer/slicer.go

@@ -16,6 +16,7 @@ import (
 	"github.com/klauspost/compress/zstd"
 
 	"github.com/canonical/chisel/internal/archive"
+	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/manifestutil"
 	"github.com/canonical/chisel/internal/scripts"
@@ -146,7 +147,7 @@ func Run(options *RunOptions) error {
 	}
 
 	// Fetch all packages, using the selection order.
-	packages := make(map[string]io.ReadSeekCloser)
+	packages := make(map[string]tarball.OpenTarFunc)
 	var pkgInfos []manifestutil.PackageInfo
 	for _, slice := range options.Selection.Slices {
 		if packages[slice.Package] != nil {
@@ -158,7 +159,7 @@ func Run(options *RunOptions) error {
 			return err
 		}
 		defer reader.Close()
-		packages[slice.Package] = reader
+		packages[slice.Package] = deb.OpenTar(reader)
 		pkgInfos = append(pkgInfos, info)
 	}
 
@@ -235,17 +236,16 @@ func Run(options *RunOptions) error {
 
 	// Extract all packages, also using the selection order.
 	for _, slice := range options.Selection.Slices {
-		reader := packages[slice.Package]
-		if reader == nil {
+		openTar := packages[slice.Package]
+		if openTar == nil {
 			continue
 		}
-		err := tarball.Extract(reader, &tarball.ExtractOptions{
+		err := tarball.Extract(openTar, &tarball.ExtractOptions{
 			Package:   slice.Package,
 			Extract:   extract[slice.Package],
 			TargetDir: targetDir,
 			Create:    create,
 		})
-		reader.Close()
 		packages[slice.Package] = nil
 		if err != nil {
 			return err

internal/tarball/extract.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/strdist"
 )
@@ -58,7 +57,11 @@ func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) {
 	return options, nil
 }
 
-func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) {
+// OpenTarFunc opens a fresh reader over a package's inner tar stream.
+// Each call must return a stream from its start.
+type OpenTarFunc func() (io.ReadCloser, error)
+
+func Extract(openTar OpenTarFunc, options *ExtractOptions) (err error) {
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err)
@@ -79,15 +82,15 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) {
 		return err
 	}
 
-	return extractData(pkgReader, validOpts)
+	return extractEntries(openTar, validOpts)
 }
 
-func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error {
-	dataReader, err := deb.DataReader(pkgReader)
+func extractEntries(openTar OpenTarFunc, options *ExtractOptions) error {
+	tarStream, err := openTar()
 	if err != nil {
 		return err
 	}
-	defer dataReader.Close()
+	defer tarStream.Close()
 
 	oldUmask := syscall.Umask(0)
 	defer func() {
@@ -117,10 +120,10 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error {
 	// create them with the permissions defined in the tarball.
 	//
 	// The assumption is that the tar entries of the parent directories appear
-	// before the entry for the file itself. This is the case for .deb files but
-	// not for all tarballs.
+	// before the entry for the file itself. This is the case for the tarballs
+	// produced by common packaging tools but not for all tarballs.
 	tarDirMode := make(map[string]fs.FileMode)
-	tarReader := tar.NewReader(dataReader)
+	tarReader := tar.NewReader(tarStream)
 	for {
 		tarHeader, err := tarReader.Next()
 		if err == io.EOF {
@@ -261,11 +264,7 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error {
 			ExtractOptions: options,
 			pendingLinks:   pendingHardLinks,
 		}
-		_, err := pkgReader.Seek(0, io.SeekStart)
-		if err != nil {
-			return err
-		}
-		err = extractHardLinks(pkgReader, extractHardLinkOptions)
+		err := extractHardLinks(openTar, extractHardLinkOptions)
 		if err != nil {
 			return err
 		}
@@ -299,14 +298,14 @@ type extractHardLinkOptions struct {
 
 // extractHardLinks iterates through the tarball a second time to extract the
 // hard links that were not extracted in the first pass.
-func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error {
-	dataReader, err := deb.DataReader(pkgReader)
+func extractHardLinks(openTar OpenTarFunc, opts *extractHardLinkOptions) error {
+	tarStream, err := openTar()
 	if err != nil {
 		return err
 	}
-	defer dataReader.Close()
+	defer tarStream.Close()
 
-	tarReader := tar.NewReader(dataReader)
+	tarReader := tar.NewReader(tarStream)
 	for {
 		tarHeader, err := tarReader.Next()
 		if err == io.EOF {