Skip to content

Commit 816587a

Browse files
zimbatmzimbatm
committed
yo
1 parent 5f39b64 commit 816587a

File tree

9 files changed

+720
-82
lines changed

9 files changed

+720
-82
lines changed

‎pkg/fastnar/README.md‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Fast NAR Reader (fastnar)
2+
3+
This package provides a high-performance NAR (Nix Archive) reader implementation, thanks to the genius brain of @edef. It improves upon the original `pkg/nar` package with a synchronous, state-machine based approach.
4+
5+
### Performance Improvements
6+
7+
1. **Synchronous Processing**: No goroutines or channels overhead
8+
2. **Buffered Reading**: Uses `bufio.Reader` with peek/discard operations
9+
3. **Pre-computed Tokens**: Binary token matching for faster parsing
10+
4. **Lower Memory Allocation**: Minimal object creation during parsing
11+
5. **State Machine**: Direct state transitions without intermediate objects
12+
13+
## Usage Examples
14+
15+
### Basic Usage
16+
17+
```go
18+
package main
19+
20+
import (
21+
"fmt"
22+
"io"
23+
"os"
24+
25+
"github.com/nix-community/go-nix/pkg/fastnar"
26+
)
27+
28+
func main() {
29+
file, err := os.Open("archive.nar")
30+
if err != nil {
31+
panic(err)
32+
}
33+
defer file.Close()
34+
35+
reader := nar.NewReader(file)
36+
37+
for {
38+
tag, err := reader.Next()
39+
if err == io.EOF {
40+
break
41+
}
42+
if err != nil {
43+
panic(err)
44+
}
45+
46+
switch tag {
47+
case nar.TagDir:
48+
fmt.Printf("Directory: %s\n", reader.Path())
49+
case nar.TagReg:
50+
fmt.Printf("File: %s (%d bytes)\n", reader.Path(), reader.Size())
51+
case nar.TagExe:
52+
fmt.Printf("Executable: %s (%d bytes)\n", reader.Path(), reader.Size())
53+
case nar.TagSym:
54+
fmt.Printf("Symlink: %s -> %s\n", reader.Path(), reader.Target())
55+
}
56+
}
57+
}
58+
```
59+
60+
### Copying NAR Archives
61+
62+
```go
63+
// High-performance NAR copying
64+
func copyNAR(dst io.Writer, src io.Reader) error {
65+
reader := nar.NewReader(src)
66+
writer := nar.NewWriter(dst)
67+
return nar.Copy(writer, reader)
68+
}
69+
```
70+
71+
### Reading File Contents
72+
73+
```go
74+
for {
75+
tag, err := reader.Next()
76+
if err == io.EOF {
77+
break
78+
}
79+
if err != nil {
80+
return err
81+
}
82+
83+
if tag == nar.TagReg || tag == nar.TagExe {
84+
// Read file content
85+
content := make([]byte, reader.Size())
86+
_, err := io.ReadFull(reader, content)
87+
if err != nil {
88+
return err
89+
}
90+
91+
fmt.Printf("File %s: %s\n", reader.Path(), string(content))
92+
}
93+
}
94+
```
95+
96+
## Migration Guide
97+
98+
### From pkg/nar to pkg/fastnar
99+
100+
**Before:**
101+
```go
102+
import "github.com/nix-community/go-nix/pkg/nar"
103+
104+
reader, err := nar.NewReader(file)
105+
if err != nil {
106+
return err
107+
}
108+
defer reader.Close()
109+
110+
for {
111+
header, err := reader.Next()
112+
if err == io.EOF {
113+
break
114+
}
115+
if err != nil {
116+
return err
117+
}
118+
119+
switch header.Type {
120+
case nar.TypeDirectory:
121+
fmt.Printf("Dir: %s\n", header.Path)
122+
case nar.TypeRegular:
123+
fmt.Printf("File: %s\n", header.Path)
124+
if header.Executable {
125+
fmt.Printf(" (executable)\n")
126+
}
127+
case nar.TypeSymlink:
128+
fmt.Printf("Link: %s -> %s\n", header.Path, header.LinkTarget)
129+
}
130+
}
131+
```
132+
133+
**After:**
134+
```go
135+
import "github.com/nix-community/go-nix/pkg/fastnar"
136+
137+
reader := nar.NewReader(file)
138+
139+
for {
140+
tag, err := reader.Next()
141+
if err == io.EOF {
142+
break
143+
}
144+
if err != nil {
145+
return err
146+
}
147+
148+
switch tag {
149+
case nar.TagDir:
150+
fmt.Printf("Dir: %s\n", reader.Path())
151+
case nar.TagReg:
152+
fmt.Printf("File: %s\n", reader.Path())
153+
case nar.TagExe:
154+
fmt.Printf("File: %s\n", reader.Path())
155+
fmt.Printf(" (executable)\n")
156+
case nar.TagSym:
157+
fmt.Printf("Link: %s -> %s\n", reader.Path(), reader.Target())
158+
}
159+
}
160+
```

‎pkg/narv2/copy.go‎ renamed to ‎pkg/fastnar/copy.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ func copyNAR(dst Writer, src Reader, tag Tag) error {
3939
if err := dst.Entry(src.Name()); err != nil {
4040
return err
4141
}
42-
copyNAR(dst, src, tag)
42+
if err := copyNAR(dst, src, tag); err != nil {
43+
return err
44+
}
4345
}
4446
}
4547
}

‎pkg/narv2/copy_test.go‎ renamed to ‎pkg/fastnar/copy_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import (
1010
)
1111

1212
func TestRoundtrip(t *testing.T) {
13-
f, err := os.Open("../signatory.nar")
13+
// Use the test data file that exists
14+
f, err := os.Open("../../test/testdata/nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar")
1415
if err != nil {
1516
t.Fatal(err)
1617
}

‎pkg/narv2/default.nix‎ renamed to ‎pkg/fastnar/default.nix‎

File renamed without changes.

‎pkg/fastnar/example_test.go‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
package nar
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"io"
7+
"log"
8+
9+
oldnar "github.com/nix-community/go-nix/pkg/nar"
10+
)
11+
12+
func ExampleReader() {
13+
// Create a simple NAR for demonstration
14+
var buf bytes.Buffer
15+
w := NewWriter(&buf)
16+
17+
// Build a simple directory structure
18+
w.Directory()
19+
20+
w.Entry("file.txt")
21+
w.File(false, 5)
22+
w.Write([]byte("hello"))
23+
w.Close()
24+
25+
w.Entry("script.sh")
26+
w.File(true, 11)
27+
w.Write([]byte("#!/bin/bash"))
28+
w.Close()
29+
30+
w.Entry("link")
31+
w.Link("file.txt")
32+
33+
w.Close() // Close root directory
34+
35+
narData := buf.Bytes()
36+
37+
// Read with Reader
38+
fmt.Println("=== FastReader ===")
39+
r := NewReader(bytes.NewReader(narData))
40+
41+
for {
42+
tag, err := r.Next()
43+
if err == io.EOF {
44+
break
45+
}
46+
if err != nil {
47+
log.Fatal(err)
48+
}
49+
50+
switch tag {
51+
case TagDir:
52+
fmt.Printf("Directory: %s\n", r.Path())
53+
case TagReg:
54+
fmt.Printf("Regular file: %s (size: %d)\n", r.Path(), r.Size())
55+
// Read file content
56+
content := make([]byte, r.Size())
57+
io.ReadFull(r, content)
58+
fmt.Printf(" Content: %s\n", string(content))
59+
case TagExe:
60+
fmt.Printf("Executable: %s (size: %d)\n", r.Path(), r.Size())
61+
// Read file content
62+
content := make([]byte, r.Size())
63+
io.ReadFull(r, content)
64+
fmt.Printf(" Content: %s\n", string(content))
65+
case TagSym:
66+
fmt.Printf("Symlink: %s -> %s\n", r.Path(), r.Target())
67+
}
68+
}
69+
70+
// Read with traditional NAR reader for comparison
71+
fmt.Println("\n=== Traditional Reader ===")
72+
oldReader, err := oldnar.NewReader(bytes.NewReader(narData))
73+
if err != nil {
74+
log.Fatal(err)
75+
}
76+
defer oldReader.Close()
77+
78+
for {
79+
hdr, err := oldReader.Next()
80+
if err == io.EOF {
81+
break
82+
}
83+
if err != nil {
84+
log.Fatal(err)
85+
}
86+
87+
switch hdr.Type {
88+
case oldnar.TypeDirectory:
89+
fmt.Printf("Directory: %s\n", hdr.Path)
90+
case oldnar.TypeRegular:
91+
if hdr.Executable {
92+
fmt.Printf("Executable: %s (size: %d)\n", hdr.Path, hdr.Size)
93+
} else {
94+
fmt.Printf("Regular file: %s (size: %d)\n", hdr.Path, hdr.Size)
95+
}
96+
// Read file content
97+
if hdr.Size > 0 {
98+
content := make([]byte, hdr.Size)
99+
io.ReadFull(oldReader, content)
100+
fmt.Printf(" Content: %s\n", string(content))
101+
}
102+
case oldnar.TypeSymlink:
103+
fmt.Printf("Symlink: %s -> %s\n", hdr.Path, hdr.LinkTarget)
104+
}
105+
}
106+
107+
// Output:
108+
// === Reader ===
109+
// Directory: /
110+
// Regular file: /file.txt (size: 5)
111+
// Content: hello
112+
// Executable: /script.sh (size: 11)
113+
// Content: #!/bin/bash
114+
// Symlink: /link -> file.txt
115+
//
116+
// === Traditional Reader ===
117+
// Directory: /
118+
// Regular file: /file.txt (size: 5)
119+
// Content: hello
120+
// Executable: /script.sh (size: 11)
121+
// Content: #!/bin/bash
122+
// Symlink: /link -> file.txt
123+
}
124+
125+
func ExampleReader_performance() {
126+
// Performance-focused usage example
127+
var buf bytes.Buffer
128+
w := NewWriter(&buf)
129+
130+
// Create a larger directory structure
131+
w.Directory()
132+
for i := 0; i < 100; i++ {
133+
w.Entry(fmt.Sprintf("file%d.txt", i))
134+
w.File(false, 10)
135+
w.Write([]byte(fmt.Sprintf("content%03d", i)))
136+
w.Close()
137+
}
138+
w.Close()
139+
140+
narData := buf.Bytes()
141+
142+
// Reader - synchronous, low overhead
143+
fmt.Println("=== Reader (synchronous) ===")
144+
r := NewReader(bytes.NewReader(narData))
145+
146+
fileCount := 0
147+
totalSize := uint64(0)
148+
149+
for {
150+
tag, err := r.Next()
151+
if err == io.EOF {
152+
break
153+
}
154+
if err != nil {
155+
log.Fatal(err)
156+
}
157+
158+
if tag == TagReg || tag == TagExe {
159+
fileCount++
160+
totalSize += r.Size()
161+
// Skip reading content for performance
162+
io.Copy(io.Discard, r)
163+
}
164+
}
165+
166+
fmt.Printf("Files processed: %d\n", fileCount)
167+
fmt.Printf("Total size: %d bytes\n", totalSize)
168+
169+
// Output:
170+
// === Reader (synchronous) ===
171+
// Files processed: 100
172+
// Total size: 1000 bytes
173+
}

0 commit comments

Comments
 (0)