pkg/store: init

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/alecthomas/kong v0.5.0
 	github.com/dgraph-io/badger/v3 v3.2103.2
 	github.com/google/go-cmp v0.5.5
+	github.com/poolpOrg/go-fastcdc v0.0.0-20211130135149-aa8a1e8a10db
 	github.com/stretchr/testify v1.7.0
 )
 
@@ -22,10 +23,18 @@ require (
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-multihash v0.2.0 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	go.opencensus.io v0.22.5 // indirect
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
-	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	lukechampine.com/blake3 v1.1.6 // indirect
 )

go.sum

@@ -52,21 +52,34 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
 github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-multihash v0.2.0 h1:oytJb9ZA1OUW0r0f9ea18GiaPOo4SXyc7p2movyUuo4=
+github.com/multiformats/go-multihash v0.2.0/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
+github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
+github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/poolpOrg/go-fastcdc v0.0.0-20211130135149-aa8a1e8a10db h1:PfgdUkbymefXsGoYrYjCpZh1PcmQ3tDHKGt1fSDm1+o=
+github.com/poolpOrg/go-fastcdc v0.0.0-20211130135149-aa8a1e8a10db/go.mod h1:9KpZcfiu0ZuQsRGTJ8ggDEjKUndlV6TFf8IMKNhW1qA=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
@@ -93,6 +106,8 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -108,6 +123,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -123,6 +140,8 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -152,3 +171,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c=
+lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=

pkg/hash/writer.go

@@ -0,0 +1,57 @@
+package hash
+
+import (
+	"fmt"
+	"hash"
+	"io"
+)
+
+var _ io.Writer = &Writer{}
+
+// Writer is a io.Writer. Bytes can be written to it.
+// After each write, it'll allow querying for the total number of bytes written,
+// as well as the digest of the bytes written.
+// TODO: tests!
+type Writer struct {
+	h            hash.Hash
+	bytesWritten uint64
+}
+
+// Write writes the given bytes to the internal hasher
+// and increments the number of bytes written.
+func (hw *Writer) Write(p []byte) (int, error) {
+	n, err := hw.h.Write(p)
+	if err != nil {
+		return 0, fmt.Errorf("unable to write to hash function: %w", err)
+	}
+
+	hw.bytesWritten += uint64(n)
+
+	return n, nil
+}
+
+// Digest returns the digest of the internal hash function.
+func (hw *Writer) Digest() []byte {
+	return hw.h.Sum(nil)
+}
+
+// BytesWritten returns the number of bytes written.
+func (hw *Writer) BytesWritten() uint64 {
+	return hw.bytesWritten
+}
+
+// Reset wipes all internal state.
+func (hw *Writer) Reset() {
+	hw.h.Reset()
+	hw.bytesWritten = 0
+}
+
+// NewWriter returns a new hash.Writer for a given HashType.
+func NewWriter(hashType HashType) (*Writer, error) {
+	hashFunc := hashFunc(hashType)
+
+	return &Writer{
+		h:            hashFunc.New(),
+		bytesWritten: 0,
+	}, nil
+}

pkg/nixpath/chunker/chunker_test.go

@@ -0,0 +1,83 @@
+package chunker_test
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/nix-community/go-nix/pkg/nixpath/chunker"
+	"github.com/stretchr/testify/assert"
+)
+
+//go:embed simple.go
+var testData []byte
+
+// nolint:gochecknoglobals
+var chunkers = []struct {
+	Name string
+	New  func([]byte) chunker.Chunker
+}{
+	{
+		"Simple",
+		func(data []byte) chunker.Chunker {
+			return chunker.NewSimpleChunker(bytes.NewReader(data))
+		},
+	},
+	{
+		"FastCDC",
+		func(data []byte) chunker.Chunker {
+			c, err := chunker.NewFastCDCChunker(bytes.NewReader(data))
+			if err != nil {
+				panic(err)
+			}
+
+			return c
+		},
+	},
+}
+
+func TestEmptySlice(t *testing.T) {
+	for _, chunker := range chunkers {
+		t.Run(chunker.Name, func(t *testing.T) {
+			// create a new chunker with the testData
+			c := chunker.New([]byte{})
+
+			_, err := c.Next()
+			if assert.Error(t, err, "c.Next should return an error") {
+				assert.ErrorIs(t, err, io.EOF, "it should be EOF")
+			}
+		})
+	}
+}
+
+func TestSimple(t *testing.T) {
+	for _, chunker := range chunkers {
+		// grab data out of the chunker.
+		// Ensure it matches testData.
+		t.Run(chunker.Name, func(t *testing.T) {
+			// create a new chunker with the testData
+			c := chunker.New(testData)
+
+			var receivedData bytes.Buffer
+
+			for {
+				chunk, err := c.Next()
+				if err != nil {
+					if errors.Is(err, io.EOF) {
+						break
+					}
+					assert.NoError(t, err, "no other error other than EOF is accepted")
+				}
+				// write the data into the receivedData buffer
+				if _, err := receivedData.Write(chunk); err != nil {
+					panic(err)
+				}
+			}
+
+			// compare received chunk contents with what was passed into the chunker
+			assert.Equal(t, testData, receivedData.Bytes())
+		})
+	}
+}

pkg/nixpath/chunker/fastcdc.go

@@ -0,0 +1,45 @@
+package chunker
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	fastcdc "github.com/poolpOrg/go-fastcdc"
+)
+
+func NewFastCDCChunker(r io.Reader) (Chunker, error) { // nolint:ireturn
+	fastcdc.NewChunkerOptions()
+	chunkerOpts := fastcdc.NewChunkerOptions()
+
+	// FUTUREWORK: Test with different chunk sizes
+	chunkerOpts.NormalSize = 64 * 2024
+	chunkerOpts.MinSize = chunkerOpts.NormalSize / 4
+	chunkerOpts.MaxSize = chunkerOpts.NormalSize * 4
+
+	c, err := fastcdc.NewChunker(r, chunkerOpts)
+	if err != nil {
+		return nil, fmt.Errorf("unable to initialize fastcdc: %w", err)
+	}
+
+	return &FastCDCChunker{
+		c: c,
+	}, nil
+}
+
+type FastCDCChunker struct {
+	c *fastcdc.Chunker
+}
+
+func (f *FastCDCChunker) Next() (Chunk, error) {
+	chunk, err := f.c.Next()
+	if err != nil {
+		if errors.Is(err, io.EOF) {
+			return nil, err
+		}
+
+		return nil, fmt.Errorf("error getting next chunk: %w", err)
+	}
+
+	return (Chunk)(chunk.Data), nil
+}

pkg/nixpath/chunker/interface.go

@@ -0,0 +1,14 @@
+package chunker
+
+type Chunk []byte
+
+// Chunker describes the interface that a given chunker needs to implement.
+// Next() is periodically called until io.EOF is encountered.
+// In case of no error, Next() returns a new chunk.
+
+// TODO: is this interface the right one, or should we add initialization
+// to the interface? Look at how it's used in pkg/store/import.go
+
+type Chunker interface {
+	Next() (Chunk, error)
+}

pkg/nixpath/chunker/simple.go

@@ -0,0 +1,42 @@
+package chunker
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+)
+
+func NewSimpleChunker(r io.Reader) Chunker { // nolint:ireturn
+	return &SimpleChunker{
+		r: r,
+	}
+}
+
+// SimpleChunker simply returns one chunk for all of the contents.
+type SimpleChunker struct {
+	r    io.Reader
+	done bool
+}
+
+func (s *SimpleChunker) Next() (Chunk, error) {
+	// if we already read everything, return io.EOF
+	if s.done {
+		return nil, io.EOF
+	}
+
+	var buf bytes.Buffer
+
+	w, err := io.Copy(&buf, s.r)
+	if err != nil {
+		return nil, fmt.Errorf("error returning from reader: %w", err)
+	}
+
+	s.done = true
+
+	// if we got passed an empty slice, return io.EOF
+	if w == 0 {
+		return nil, io.EOF
+	}
+
+	return buf.Bytes(), nil
+}