From 215ddfe577dc0723c3fa931194a8b356a0fead02 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 22 Sep 2026 08:16:23 +0100 Subject: [PATCH] Expose archive readers as io/fs filesystems --- README.md | 29 ++++ fs.go | 264 ++++++++++++++++++++++++++++++++++ fs_test.go | 415 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 708 insertions(+) create mode 100644 fs.go create mode 100644 fs_test.go diff --git a/README.md b/README.md index c0a976b..d154c76 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,35 @@ reader, _ := archives.OpenWithPrefix("pkg.tgz", f, "package/") // files are now accessible without the package/ prefix ``` +### Using io/fs + +`NewFS` adapts a reader to `fs.FS`, with `ReadDir`, `ReadFile`, and `Stat` support. It indexes metadata once and reads file contents through the reader as needed. + +```go +archiveFS, err := archives.NewFS(reader) +if err != nil { + return err +} +data, err := fs.ReadFile(archiveFS, "lib/util.js") +if err != nil { + return err +} +fmt.Println(string(data)) +if err := fs.WalkDir(archiveFS, ".", func(name string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + fmt.Println(name) + return nil +}); err != nil { + return err +} +``` + +Import `io/fs` for these functions. Paths use `/` separators and `"."` for the root, and missing parent directories appear automatically. Archive names may have leading `./` or directory trailing slashes; other invalid paths and file/directory conflicts cause `NewFS` to return `fs.ErrInvalid`. Duplicate paths use the first entry. Symlinks and other special entries appear in listings but cannot be opened. + +Keep the reader open while using the filesystem. Closing an individual file leaves the reader open. + ### Extracting to disk `ExtractAll` writes every entry under a target directory, creating it and any intermediate directories. Entry names are validated with `filepath.Localize` so absolute paths and `..` segments that would escape the target return `ErrUnsafePath` naming the offending entry. diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..b4b69cb --- /dev/null +++ b/fs.go @@ -0,0 +1,264 @@ +package archives + +import ( + "io" + "io/fs" + "path" + "slices" + "strings" + "time" +) + +const ( + fsOpNew = "newfs" + fsOpRead = "read" + fsOpReadDir = "readdir" +) + +// FS is a read-only filesystem over a Reader. The Reader must remain open +// while the filesystem and its files are in use. +type FS struct { + reader Reader + nodes map[string]*fsNode +} + +type fsNode struct { + info archiveFileInfo + children []fs.DirEntry +} + +var ( + _ fs.ReadDirFS = (*FS)(nil) + _ fs.ReadFileFS = (*FS)(nil) + _ fs.StatFS = (*FS)(nil) + _ fs.ReadDirFile = (*archiveFile)(nil) +) + +// NewFS indexes r without extracting file contents. Missing parent directories +// are created in the index. Leading "./" and directory trailing slashes are +// removed from archive names; other invalid fs paths and file/directory +// conflicts return fs.ErrInvalid. The first entry for each path is used. +// Symlinks and other special entries can be listed, but cannot be opened. +// Closing an opened file does not close r; the caller owns the Reader. +func NewFS(r Reader) (*FS, error) { + entries, err := r.List() + if err != nil { + return nil, &fs.PathError{Op: fsOpNew, Path: ".", Err: err} + } + f := &FS{reader: r, nodes: make(map[string]*fsNode, len(entries))} + for _, entry := range entries { + if err := f.addEntry(entry); err != nil { + return nil, err + } + } + if err := f.addDirectories(); err != nil { + return nil, err + } + for name, node := range f.nodes { + if name != "." { + parent := f.nodes[path.Dir(name)] + parent.children = append(parent.children, fs.FileInfoToDirEntry(node.info)) + } + } + for _, node := range f.nodes { + slices.SortFunc(node.children, func(a, b fs.DirEntry) int { + return strings.Compare(a.Name(), b.Name()) + }) + } + return f, nil +} + +func (f *FS) addEntry(entry FileInfo) error { + name := entry.Path + if entry.IsDir { + name = strings.TrimSuffix(name, "/") + } + for strings.HasPrefix(name, "./") { + name = strings.TrimPrefix(name, "./") + } + if !fs.ValidPath(name) || (name == "." && !entry.IsDir) { + return &fs.PathError{Op: fsOpNew, Path: entry.Path, Err: fs.ErrInvalid} + } + if previous, ok := f.nodes[name]; ok { + if previous.info.IsDir() != entry.IsDir { + return &fs.PathError{Op: fsOpNew, Path: entry.Path, Err: fs.ErrInvalid} + } + return nil + } + f.nodes[name] = &fsNode{info: archiveFileInfo{entry: entry, name: path.Base(name)}} + return nil +} + +func (f *FS) addDirectories() error { + if _, ok := f.nodes["."]; !ok { + f.nodes["."] = implicitDirectory(".") + } + for name := range f.nodes { + for parent := path.Dir(name); ; parent = path.Dir(parent) { + if node, ok := f.nodes[parent]; ok { + if !node.info.IsDir() { + return &fs.PathError{Op: fsOpNew, Path: parent, Err: fs.ErrInvalid} + } + break + } + f.nodes[parent] = implicitDirectory(path.Base(parent)) + } + } + return nil +} + +func implicitDirectory(name string) *fsNode { + return &fsNode{info: archiveFileInfo{name: name, entry: FileInfo{IsDir: true}}} +} + +func (f *FS) lookup(op, name string) (*fsNode, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: op, Path: name, Err: fs.ErrInvalid} + } + node, ok := f.nodes[name] + if !ok { + return nil, &fs.PathError{Op: op, Path: name, Err: fs.ErrNotExist} + } + return node, nil +} + +// Open opens a file or directory. Names follow fs.ValidPath, with "." as root. +// +//nolint:ireturn // fs.FS requires this return type +func (f *FS) Open(name string) (fs.File, error) { + node, err := f.lookup("open", name) + if err != nil { + return nil, err + } + file := &archiveFile{name: name, node: node} + if node.info.IsDir() { + return file, nil + } + if !node.info.Mode().IsRegular() { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + file.content, err = f.reader.Extract(node.info.entry.Path) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + return file, nil +} + +// Stat returns metadata for a file or directory without extracting its contents. +// +//nolint:ireturn // fs.StatFS requires this return type +func (f *FS) Stat(name string) (fs.FileInfo, error) { + node, err := f.lookup("stat", name) + if err != nil { + return nil, err + } + return node.info, nil +} + +// ReadDir returns directory entries sorted by name. +func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { + node, err := f.lookup(fsOpReadDir, name) + if err != nil { + return nil, err + } + if !node.info.IsDir() { + return nil, &fs.PathError{Op: fsOpReadDir, Path: name, Err: fs.ErrInvalid} + } + return append([]fs.DirEntry{}, node.children...), nil +} + +// ReadFile reads the named file's contents. +func (f *FS) ReadFile(name string) ([]byte, error) { + file, err := f.Open(name) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + return io.ReadAll(file) +} + +type archiveFileInfo struct { + entry FileInfo + name string +} + +func (i archiveFileInfo) Name() string { return i.name } +func (i archiveFileInfo) Size() int64 { return i.entry.Size } +func (i archiveFileInfo) ModTime() time.Time { return i.entry.ModTime } +func (i archiveFileInfo) IsDir() bool { return i.entry.IsDir } +func (i archiveFileInfo) Sys() any { return nil } + +func (i archiveFileInfo) Mode() fs.FileMode { + mode := fs.FileMode(i.entry.Mode) + if !i.entry.HasMode { + mode = mode.Type() | extractFilePerm + if i.IsDir() { + mode = mode.Type() | extractDirPerm + } + } + if i.IsDir() { + mode |= fs.ModeDir + } + return mode +} + +type archiveFile struct { + name string + node *fsNode + content io.ReadCloser + offset int + closed bool +} + +//nolint:ireturn // fs.File requires this return type +func (f *archiveFile) Stat() (fs.FileInfo, error) { + if f.closed { + return nil, &fs.PathError{Op: "stat", Path: f.name, Err: fs.ErrClosed} + } + return f.node.info, nil +} + +func (f *archiveFile) Read(b []byte) (int, error) { + if f.closed { + return 0, &fs.PathError{Op: fsOpRead, Path: f.name, Err: fs.ErrClosed} + } + if f.node.info.IsDir() { + return 0, &fs.PathError{Op: fsOpRead, Path: f.name, Err: fs.ErrInvalid} + } + n, err := f.content.Read(b) + if err != nil && err != io.EOF { + err = &fs.PathError{Op: fsOpRead, Path: f.name, Err: err} + } + return n, err +} + +func (f *archiveFile) Close() error { + if f.closed { + return &fs.PathError{Op: "close", Path: f.name, Err: fs.ErrClosed} + } + f.closed = true + if f.content != nil { + if err := f.content.Close(); err != nil { + return &fs.PathError{Op: "close", Path: f.name, Err: err} + } + } + return nil +} + +func (f *archiveFile) ReadDir(n int) ([]fs.DirEntry, error) { + if f.closed { + return nil, &fs.PathError{Op: fsOpReadDir, Path: f.name, Err: fs.ErrClosed} + } + if !f.node.info.IsDir() { + return nil, &fs.PathError{Op: fsOpReadDir, Path: f.name, Err: fs.ErrInvalid} + } + remaining := f.node.children[f.offset:] + if n > 0 && len(remaining) == 0 { + return nil, io.EOF + } + if n > 0 && n < len(remaining) { + remaining = remaining[:n] + } + f.offset += len(remaining) + return append([]fs.DirEntry{}, remaining...), nil +} diff --git a/fs_test.go b/fs_test.go new file mode 100644 index 0000000..5bf493b --- /dev/null +++ b/fs_test.go @@ -0,0 +1,415 @@ +package archives + +import ( + "archive/tar" + "archive/zip" + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "slices" + "testing" + "testing/fstest" + "time" +) + +func openTestFS(t *testing.T, name string, data []byte, prefix string) *FS { + t.Helper() + r, err := OpenBytesWithPrefix(name, data, prefix) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = r.Close() }) + f, err := NewFS(r) + if err != nil { + t.Fatal(err) + } + return f +} + +func TestFSFormats(t *testing.T) { + for _, tc := range []struct { + name string + data []byte + prefix string + files []string + }{ + {"test.zip", createTestZip(), "", []string{"README.md", "src/main.go", "src/util/helper.go", "docs/guide.md"}}, + {"test.tar.gz", createTestTarGz(), "", []string{"package.json", "index.js", "lib/util.js"}}, + {"test.gem", createTestGem(), "", []string{"lib/main.rb"}}, + {"test.conda", createTestConda(t), "", []string{"site-packages/six.py", "info/index.json", "info/paths.json", "info/licenses/LICENSE", "site-packages/six-1.0/META"}}, + {"prefix.zip", createTestZipWithDirEntries(), "project-abc123/", []string{"README.md", "src/main.go", "src/util.go"}}, + {"prefix.tgz", createTestTarGzWithDirEntries(), "project-abc123/", []string{"README.md", "src/main.go", "src/util.go"}}, + } { + t.Run(tc.name, func(t *testing.T) { + f := openTestFS(t, tc.name, tc.data, tc.prefix) + if err := fstest.TestFS(f, tc.files...); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestFSWalkAndSub(t *testing.T) { + f := openTestFS(t, "test.zip", createTestZip(), "") + var paths []string + err := fs.WalkDir(f, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + paths = append(paths, name) + return nil + }) + if err != nil { + t.Fatal(err) + } + want := []string{".", "README.md", "docs", "docs/guide.md", "src", "src/main.go", "src/util", "src/util/helper.go"} + if !slices.Equal(paths, want) { + t.Fatalf("walk = %v, want %v", paths, want) + } + sub, err := fs.Sub(f, "src") + if err != nil { + t.Fatal(err) + } + data, err := fs.ReadFile(sub, "util/helper.go") + if err != nil || string(data) != "package util" { + t.Fatalf("ReadFile = %q, %v", data, err) + } +} + +func TestFSPathErrors(t *testing.T) { + f := openTestFS(t, "test.zip", createTestZip(), "") + operations := map[string]func(string) error{ + "open": func(name string) error { + file, err := f.Open(name) + if file != nil { + _ = file.Close() + } + return err + }, + "stat": func(name string) error { _, err := fs.Stat(f, name); return err }, + "readdir": func(name string) error { _, err := fs.ReadDir(f, name); return err }, + "readfile": func(name string) error { _, err := fs.ReadFile(f, name); return err }, + } + for op, run := range operations { + t.Run(op, func(t *testing.T) { + for _, name := range []string{"", "/", "/README.md", "./README.md", "src/", "src//main.go", "src/../README.md", "../README.md", "\xff"} { + assertFSPathError(t, run(name), name, fs.ErrInvalid) + } + for _, name := range []string{"missing", "missing/file", "README.md/child"} { + assertFSPathError(t, run(name), name, fs.ErrNotExist) + } + }) + } + _, err := fs.ReadFile(f, "src") + assertFSPathError(t, err, "src", fs.ErrInvalid) + _, err = fs.ReadDir(f, "README.md") + assertFSPathError(t, err, "README.md", fs.ErrInvalid) +} + +func assertFSPathError(t *testing.T, err error, name string, want error) { + t.Helper() + var pe *fs.PathError + if !errors.Is(err, want) || !errors.As(err, &pe) || pe.Path != name { + t.Fatalf("error = %v, want PathError for %q wrapping %v", err, name, want) + } +} + +func TestFSDirectoryReads(t *testing.T) { + f := openTestFS(t, "test.zip", createTestZip(), "") + dir, err := f.Open(".") + if err != nil { + t.Fatal(err) + } + defer func() { _ = dir.Close() }() + rd := dir.(fs.ReadDirFile) + for _, want := range []string{"README.md", "docs", "src"} { + entries, err := rd.ReadDir(1) + if err != nil || len(entries) != 1 || entries[0].Name() != want { + t.Fatalf("ReadDir(1) = %v, %v, want %s", entries, err, want) + } + } + entries, err := rd.ReadDir(1) + if len(entries) != 0 || err != io.EOF { + t.Fatalf("ReadDir at EOF = %v, %v", entries, err) + } + entries, err = rd.ReadDir(0) + if entries == nil || len(entries) != 0 || err != nil { + t.Fatalf("ReadDir(0) at EOF = %v, %v", entries, err) + } + entries, err = f.ReadDir(".") + if err != nil { + t.Fatal(err) + } + entries[0] = nil + again, err := f.ReadDir(".") + if err != nil || again[0] == nil { + t.Fatalf("caller changed stored directory entries: %v, %v", again, err) + } +} + +func TestFSClosedFiles(t *testing.T) { + f := openTestFS(t, "test.zip", createTestZip(), "") + for _, name := range []string{"README.md", "."} { + file, err := f.Open(name) + if err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + _, err = file.Read(make([]byte, 1)) + assertFSPathError(t, err, name, fs.ErrClosed) + _, err = file.Stat() + assertFSPathError(t, err, name, fs.ErrClosed) + _, err = file.(fs.ReadDirFile).ReadDir(1) + assertFSPathError(t, err, name, fs.ErrClosed) + assertFSPathError(t, file.Close(), name, fs.ErrClosed) + } + data, err := fs.ReadFile(f, "README.md") + if err != nil || string(data) != "# Test Package" { + t.Fatalf("read after closing other files = %q, %v", data, err) + } +} + +func TestFSArchivePathsAndMetadata(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + modified := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + writeTarFile(t, tw, "./lib/tool", "first", 0o750) + writeTarFile(t, tw, "./lib/tool", "duplicate", 0o600) + for _, header := range []*tar.Header{ + {Name: "./", Typeflag: tar.TypeDir, Mode: 0o700, ModTime: modified}, + {Name: "./lib/", Typeflag: tar.TypeDir, Mode: 0o710, ModTime: modified}, + {Name: "empty/", Typeflag: tar.TypeDir, Mode: 0o755}, + } { + if err := tw.WriteHeader(header); err != nil { + t.Fatal(err) + } + } + writeTarFile(t, tw, " space /file", "space", 0o000) + if err := tw.Close(); err != nil { + t.Fatal(err) + } + f := openTestFS(t, "test.tar", buf.Bytes(), "") + if err := fstest.TestFS(f, "lib/tool", "empty", " space /file"); err != nil { + t.Fatal(err) + } + data, err := fs.ReadFile(f, "lib/tool") + if err != nil || string(data) != "first" { + t.Fatalf("ReadFile = %q, %v", data, err) + } + for _, tc := range []struct { + name string + mode fs.FileMode + size int64 + }{ + {".", fs.ModeDir | 0o700, 0}, + {"lib", fs.ModeDir | 0o710, 0}, + {"lib/tool", 0o750, 5}, + {" space /file", 0, 5}, + } { + info, err := fs.Stat(f, tc.name) + if err != nil { + t.Fatal(err) + } + if info.Mode() != tc.mode || info.Size() != tc.size { + t.Errorf("%s: mode/size = %v/%d, want %v/%d", tc.name, info.Mode(), info.Size(), tc.mode, tc.size) + } + if info.IsDir() && !info.ModTime().Equal(modified) { + t.Errorf("%s: ModTime = %v, want %v", tc.name, info.ModTime(), modified) + } + } +} + +func TestFSBackslashName(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + writeTarFile(t, tw, "a\\b", "backslash", 0o644) + if err := tw.Close(); err != nil { + t.Fatal(err) + } + f := openTestFS(t, "test.tar", buf.Bytes(), "") + data, err := fs.ReadFile(f, "a\\b") + if err != nil || string(data) != "backslash" { + t.Fatalf("ReadFile = %q, %v", data, err) + } + entries, err := fs.ReadDir(f, ".") + if err != nil || len(entries) != 1 || entries[0].Name() != "a\\b" { + t.Fatalf("ReadDir = %v, %v", entries, err) + } +} + +func TestFSRejectsInvalidArchivePaths(t *testing.T) { + for _, names := range [][]string{ + {"../escape"}, {"/absolute"}, {"a/../b"}, {"a//b"}, {"a/./b"}, {"."}, {".//"}, {"/"}, + {"file", "file/child"}, {"file/child", "file"}, {"file", "file/"}, + } { + t.Run(fmt.Sprint(names), func(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, name := range names { + if _, err := zw.Create(name); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + r, err := OpenBytes("test.zip", buf.Bytes()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + if _, err := NewFS(r); !errors.Is(err, fs.ErrInvalid) { + t.Fatalf("NewFS = %v, want ErrInvalid", err) + } + }) + } +} + +func TestFSEmptyArchive(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + if err := zw.Close(); err != nil { + t.Fatal(err) + } + f := openTestFS(t, "empty.zip", buf.Bytes(), "") + if err := fstest.TestFS(f); err != nil { + t.Fatal(err) + } +} + +func TestFSSpecialEntries(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, entry := range []struct { + name string + kind byte + }{ + {"symlink", tar.TypeSymlink}, {"hardlink", tar.TypeLink}, {"fifo", tar.TypeFifo}, + } { + if err := tw.WriteHeader(&tar.Header{Name: entry.name, Typeflag: entry.kind, Linkname: "target", Mode: 0o644}); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + f := openTestFS(t, "special.tar", buf.Bytes(), "") + entries, err := fs.ReadDir(f, ".") + if err != nil || len(entries) != 3 { + t.Fatalf("ReadDir = %v, %v", entries, err) + } + for _, entry := range entries { + if entry.Type().IsRegular() { + t.Errorf("%s reported as regular", entry.Name()) + } + _, err := f.Open(entry.Name()) + assertFSPathError(t, err, entry.Name(), fs.ErrInvalid) + } +} + +type fsErrorReader struct { + Reader + listErr error + extractErr error + content io.ReadCloser +} + +func (r *fsErrorReader) List() ([]FileInfo, error) { + if r.listErr != nil { + return nil, r.listErr + } + return r.Reader.List() +} + +func (r *fsErrorReader) Extract(name string) (io.ReadCloser, error) { + return r.content, r.extractErr +} + +type fsErrorContent struct { + err error + closed bool +} + +func (c *fsErrorContent) Read(b []byte) (int, error) { return 0, c.err } +func (c *fsErrorContent) Close() error { + c.closed = true + return nil +} + +func TestFSReaderErrors(t *testing.T) { + r, err := OpenBytes("test.zip", createTestZip()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + want := errors.New("reader failure") + wrapped := &fsErrorReader{Reader: r, listErr: want} + _, err = NewFS(wrapped) + assertFSPathError(t, err, ".", want) + wrapped.listErr = nil + wrapped.extractErr = want + f, err := NewFS(wrapped) + if err != nil { + t.Fatal(err) + } + if _, err := fs.Stat(f, "README.md"); err != nil { + t.Fatalf("Stat extracted contents: %v", err) + } + _, err = fs.ReadFile(f, "README.md") + assertFSPathError(t, err, "README.md", want) + content := &fsErrorContent{err: want} + wrapped.extractErr = nil + wrapped.content = content + _, err = fs.ReadFile(f, "README.md") + assertFSPathError(t, err, "README.md", want) + if !content.closed { + t.Fatal("ReadFile did not close the extracted stream after a read error") + } +} + +func ExampleNewFS() { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("lib/hello.txt") + if err != nil { + panic(err) + } + if _, err := io.WriteString(w, "hello\n"); err != nil { + panic(err) + } + if err := zw.Close(); err != nil { + panic(err) + } + r, err := Open("package.zip", &buf) + if err != nil { + panic(err) + } + defer func() { _ = r.Close() }() + f, err := NewFS(r) + if err != nil { + panic(err) + } + err = fs.WalkDir(f, ".", func(name string, entry fs.DirEntry, err error) error { + if err == nil { + fmt.Println(name) + } + return err + }) + if err != nil { + panic(err) + } + data, err := fs.ReadFile(f, "lib/hello.txt") + if err != nil { + panic(err) + } + fmt.Print(string(data)) + // Output: + // . + // lib + // lib/hello.txt + // hello +}