Faster nar parser

pkg/narv2/README.md

@@ -0,0 +1,160 @@
+# Fast NAR Reader (fastnar)
+
+This package provides a high-performance NAR (Nix Archive) reader implementation, thanks to the genius brain of @edef. It improves upon the original `pkg/nar` package with a synchronous, state-machine based approach.
+
+### Performance Improvements
+
+1. **Synchronous Processing**: No goroutines or channels overhead
+2. **Buffered Reading**: Uses `bufio.Reader` with peek/discard operations
+3. **Pre-computed Tokens**: Binary token matching for faster parsing
+4. **Lower Memory Allocation**: Minimal object creation during parsing
+5. **State Machine**: Direct state transitions without intermediate objects
+
+## Usage Examples
+
+### Basic Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "io"
+    "os"
+
+    "github.com/nix-community/go-nix/pkg/fastnar"
+)
+
+func main() {
+    file, err := os.Open("archive.nar")
+    if err != nil {
+        panic(err)
+    }
+    defer file.Close()
+
+    reader := nar.NewReader(file)
+
+    for {
+        tag, err := reader.Next()
+        if err == io.EOF {
+            break
+        }
+        if err != nil {
+            panic(err)
+        }
+
+        switch tag {
+        case nar.TagDir:
+            fmt.Printf("Directory: %s\n", reader.Path())
+        case nar.TagReg:
+            fmt.Printf("File: %s (%d bytes)\n", reader.Path(), reader.Size())
+        case nar.TagExe:
+            fmt.Printf("Executable: %s (%d bytes)\n", reader.Path(), reader.Size())
+        case nar.TagSym:
+            fmt.Printf("Symlink: %s -> %s\n", reader.Path(), reader.Target())
+        }
+    }
+}
+```
+
+### Copying NAR Archives
+
+```go
+// High-performance NAR copying
+func copyNAR(dst io.Writer, src io.Reader) error {
+    reader := nar.NewReader(src)
+    writer := nar.NewWriter(dst)
+    return nar.Copy(writer, reader)
+}
+```
+
+### Reading File Contents
+
+```go
+for {
+    tag, err := reader.Next()
+    if err == io.EOF {
+        break
+    }
+    if err != nil {
+        return err
+    }
+
+    if tag == nar.TagReg || tag == nar.TagExe {
+        // Read file content
+        content := make([]byte, reader.Size())
+        _, err := io.ReadFull(reader, content)
+        if err != nil {
+            return err
+        }
+
+        fmt.Printf("File %s: %s\n", reader.Path(), string(content))
+    }
+}
+```
+
+## Migration Guide
+
+### From pkg/nar to pkg/fastnar
+
+**Before:**
+```go
+import "github.com/nix-community/go-nix/pkg/nar"
+
+reader, err := nar.NewReader(file)
+if err != nil {
+    return err
+}
+defer reader.Close()
+
+for {
+    header, err := reader.Next()
+    if err == io.EOF {
+        break
+    }
+    if err != nil {
+        return err
+    }
+
+    switch header.Type {
+    case nar.TypeDirectory:
+        fmt.Printf("Dir: %s\n", header.Path)
+    case nar.TypeRegular:
+        fmt.Printf("File: %s\n", header.Path)
+        if header.Executable {
+            fmt.Printf("  (executable)\n")
+        }
+    case nar.TypeSymlink:
+        fmt.Printf("Link: %s -> %s\n", header.Path, header.LinkTarget)
+    }
+}
+```
+
+**After:**
+```go
+import "github.com/nix-community/go-nix/pkg/fastnar"
+
+reader := nar.NewReader(file)
+
+for {
+    tag, err := reader.Next()
+    if err == io.EOF {
+        break
+    }
+    if err != nil {
+        return err
+    }
+
+    switch tag {
+    case nar.TagDir:
+        fmt.Printf("Dir: %s\n", reader.Path())
+    case nar.TagReg:
+        fmt.Printf("File: %s\n", reader.Path())
+    case nar.TagExe:
+        fmt.Printf("File: %s\n", reader.Path())
+        fmt.Printf("  (executable)\n")
+    case nar.TagSym:
+        fmt.Printf("Link: %s -> %s\n", reader.Path(), reader.Target())
+    }
+}
+```

pkg/narv2/copy.go

@@ -0,0 +1,47 @@
+package narv2
+
+import "io"
+
+func Copy(dst Writer, src Reader) error {
+	tag, err := src.Next()
+	if err != nil {
+		return err
+	}
+	return copyNAR(dst, src, tag)
+}
+
+func copyNAR(dst Writer, src Reader, tag Tag) error {
+	switch tag {
+	default:
+		panic("invalid tag")
+	case TagSym:
+		return dst.Link(src.Target())
+	case TagReg, TagExe:
+		if err := dst.File(tag == TagExe, src.Size()); err != nil {
+			return err
+		}
+		if _, err := io.Copy(dst, src); err != nil {
+			return err
+		}
+		return dst.Close()
+	case TagDir:
+		if err := dst.Directory(); err != nil {
+			return err
+		}
+		for {
+			tag, err := src.Next()
+			if err == io.EOF {
+				return dst.Close()
+			}
+			if err != nil {
+				return err
+			}
+			if err := dst.Entry(src.Name()); err != nil {
+				return err
+			}
+			if err := copyNAR(dst, src, tag); err != nil {
+				return err
+			}
+		}
+	}
+}

pkg/narv2/copy_test.go

@@ -0,0 +1,88 @@
+package narv2_test
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/nix-community/go-nix/pkg/narv2"
+)
+
+func TestRoundtrip(t *testing.T) {
+	// Use the test data file that exists
+	f, err := os.Open("../../test/testdata/nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	// Read the original file into memory
+	original, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Copy original through narv2
+	var outputBuf bytes.Buffer
+	if err := narv2.Copy(narv2.NewWriter(&outputBuf), narv2.NewReader(bytes.NewReader(original))); err != nil {
+		t.Fatalf("Copy: %v", err)
+	}
+
+	// Test logical equivalence: read both files and compare their structure
+	originalEntries := readAllEntries(t, bytes.NewReader(original))
+	outputEntries := readAllEntries(t, bytes.NewReader(outputBuf.Bytes()))
+
+	if len(originalEntries) != len(outputEntries) {
+		t.Fatalf("Entry count mismatch: original=%d, output=%d", len(originalEntries), len(outputEntries))
+	}
+
+	for i, orig := range originalEntries {
+		out := outputEntries[i]
+		if orig.Path != out.Path || orig.Type != out.Type || orig.Size != out.Size || orig.Target != out.Target {
+			t.Errorf("Entry %d mismatch:\n  original: %+v\n  output:   %+v", i, orig, out)
+		}
+	}
+}
+
+type EntryInfo struct {
+	Path   string
+	Type   string
+	Size   uint64
+	Target string
+}
+
+func readAllEntries(t *testing.T, r io.Reader) []EntryInfo {
+	var entries []EntryInfo
+	reader := narv2.NewReader(r)
+
+	for {
+		tag, err := reader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("Reader error: %v", err)
+		}
+
+		entry := EntryInfo{Path: reader.Path()}
+		switch tag {
+		case narv2.TagDir:
+			entry.Type = "directory"
+		case narv2.TagReg:
+			entry.Type = "regular"
+			entry.Size = reader.Size()
+			io.Copy(io.Discard, reader) // consume content
+		case narv2.TagExe:
+			entry.Type = "executable"
+			entry.Size = reader.Size()
+			io.Copy(io.Discard, reader) // consume content
+		case narv2.TagSym:
+			entry.Type = "symlink"
+			entry.Target = reader.Target()
+		}
+		entries = append(entries, entry)
+	}
+
+	return entries
+}