Skip to content

Commit 19eb4ca

Browse files
brianmcgeebrianmcgee
committed
feat: add sqlite helper and some integration tests
1 parent 9ab4739 commit 19eb4ca

File tree

8 files changed

+220
-52
lines changed

8 files changed

+220
-52
lines changed

‎go.mod‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ module github.com/nix-community/go-nix
33
go 1.20
44

55
require (
6+
github.com/adrg/xdg v0.5.0
67
github.com/alecthomas/kong v0.5.0
78
github.com/dgraph-io/badger/v3 v3.2103.2
9+
github.com/mattn/go-sqlite3 v1.14.23
810
github.com/multiformats/go-multihash v0.2.1
911
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
10-
github.com/stretchr/testify v1.7.0
12+
github.com/stretchr/testify v1.9.0
1113
)
1214

1315
require (
@@ -24,7 +26,6 @@ require (
2426
github.com/google/flatbuffers v1.12.1 // indirect
2527
github.com/klauspost/compress v1.12.3 // indirect
2628
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
27-
github.com/mattn/go-sqlite3 v1.14.23 // indirect
2829
github.com/minio/sha256-simd v1.0.0 // indirect
2930
github.com/mr-tron/base58 v1.2.0 // indirect
3031
github.com/multiformats/go-varint v0.0.6 // indirect
@@ -34,7 +35,7 @@ require (
3435
go.opencensus.io v0.22.5 // indirect
3536
golang.org/x/crypto v0.17.0 // indirect
3637
golang.org/x/net v0.17.0 // indirect
37-
golang.org/x/sys v0.15.0 // indirect
38-
gopkg.in/yaml.v3 v3.0.0 // indirect
38+
golang.org/x/sys v0.22.0 // indirect
39+
gopkg.in/yaml.v3 v3.0.1 // indirect
3940
lukechampine.com/blake3 v1.1.6 // indirect
4041
)

‎go.sum‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
22
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
33
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
44
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
5+
github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY=
6+
github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4=
57
github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE=
68
github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
79
github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48=
@@ -91,8 +93,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
9193
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
9294
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
9395
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
94-
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
9596
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
97+
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
98+
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
9699
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
97100
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
98101
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -135,8 +138,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
135138
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
136139
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
137140
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
138-
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
139-
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
141+
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
142+
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
140143
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
141144
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
142145
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -162,8 +165,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
162165
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
163166
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
164167
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
165-
gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
166-
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
168+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
169+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
167170
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
168171
lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c=
169172
lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=

‎pkg/sqlite/binary_cache_v6/query.sql‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ on conflict (url)
55
do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5
66
returning id;
77

8-
-- todo should this be :many?
9-
-- name: QueryCache :one
8+
-- name: QueryCache :many
109
select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?;
1110

1211
-- name: InsertNar :exec
@@ -18,8 +17,7 @@ insert or replace into NARs(
1817
-- name: InsertMissingNAR :exec
1918
insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0);
2019

21-
-- todo should this be :many?
22-
-- name: QueryNar :one
20+
-- name: QueryNar :many
2321
select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs
2422
where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?));
2523

‎pkg/sqlite/binary_cache_v6/query.sql.go‎

Lines changed: 62 additions & 32 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/sqlite/fetcher_cache_v2/query.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- name: UpsertCache :exec
22
insert or replace into Cache(domain, key, value, timestamp) values (?, ?, ?, ?);
33

4-
-- name: QueryCache :one
4+
-- name: QueryCache :many
55
select value, timestamp from Cache where domain = ? and key = ?;

‎pkg/sqlite/fetcher_cache_v2/query.sql.go‎

Lines changed: 22 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/sqlite/sqlite.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package sqlite
2+
3+
import (
4+
"database/sql"
5+
"fmt"
6+
7+
// enable the sqlite3 driver.
8+
_ "github.com/mattn/go-sqlite3"
9+
"github.com/nix-community/go-nix/pkg/sqlite/binary_cache_v6"
10+
"github.com/nix-community/go-nix/pkg/sqlite/eval_cache_v5"
11+
"github.com/nix-community/go-nix/pkg/sqlite/fetcher_cache_v2"
12+
"github.com/nix-community/go-nix/pkg/sqlite/nix_v10"
13+
)
14+
15+
func BinaryCacheV6(dsn string) (*sql.DB, *binary_cache_v6.Queries, error) {
16+
db, err := sql.Open("sqlite3", dsn)
17+
if err != nil {
18+
return nil, nil, fmt.Errorf("failed to open database: %w", err)
19+
}
20+
21+
return db, binary_cache_v6.New(db), nil
22+
}
23+
24+
func EvalCacheV5(dsn string) (*sql.DB, *eval_cache_v5.Queries, error) {
25+
db, err := sql.Open("sqlite3", dsn)
26+
if err != nil {
27+
return nil, nil, fmt.Errorf("failed to open database: %w", err)
28+
}
29+
30+
return db, eval_cache_v5.New(db), nil
31+
}
32+
33+
func FetcherCacheV2(dsn string) (*sql.DB, *fetcher_cache_v2.Queries, error) {
34+
db, err := sql.Open("sqlite3", dsn)
35+
if err != nil {
36+
return nil, nil, fmt.Errorf("failed to open database: %w", err)
37+
}
38+
39+
return db, fetcher_cache_v2.New(db), nil
40+
}
41+
42+
func NixV10(dsn string) (*sql.DB, *nix_v10.Queries, error) {
43+
db, err := sql.Open("sqlite3", dsn)
44+
if err != nil {
45+
return nil, nil, fmt.Errorf("failed to open database: %w", err)
46+
}
47+
48+
return db, nix_v10.New(db), nil
49+
}

‎pkg/sqlite/sqlite_test.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
//go:build integration
2+
3+
package sqlite
4+
5+
import (
6+
"context"
7+
"fmt"
8+
"os/exec"
9+
"testing"
10+
11+
"github.com/adrg/xdg"
12+
"github.com/nix-community/go-nix/pkg/sqlite/fetcher_cache_v2"
13+
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
func TestBinaryCacheV6(t *testing.T) {
18+
as := require.New(t)
19+
20+
// open our user-specific binary cache db
21+
path, err := xdg.CacheFile("nix/binary-cache-v6.sqlite")
22+
as.NoError(err, "failed to resolve binary cache file")
23+
as.FileExists(path)
24+
25+
// open the sqlite db
26+
db, queries, err := BinaryCacheV6(fmt.Sprintf("file:%s?mode=ro", path))
27+
as.NoError(err)
28+
defer db.Close()
29+
30+
// perform a basic query, we aren't interested in the result
31+
_, err = queries.QueryLastPurge(context.Background())
32+
as.NoError(err)
33+
}
34+
35+
func TestFetcherCacheV2(t *testing.T) {
36+
as := require.New(t)
37+
38+
// open our user-specific binary cache db
39+
path, err := xdg.CacheFile("nix/fetcher-cache-v2.sqlite")
40+
as.NoError(err, "failed to resolve fetcher cache file")
41+
as.FileExists(path)
42+
43+
// open the sqlite db
44+
db, queries, err := FetcherCacheV2(fmt.Sprintf("file:%s?mode=ro", path))
45+
as.NoError(err)
46+
defer db.Close()
47+
48+
// perform a basic query, we aren't interested in the result
49+
_, err = queries.QueryCache(context.Background(), fetcher_cache_v2.QueryCacheParams{})
50+
as.NoError(err)
51+
}
52+
53+
func TestNixV10(t *testing.T) {
54+
as := require.New(t)
55+
56+
// pull down a known path
57+
path := "/nix/store/kz5clxh7s1n0fnx6d37c1wc2cs9qm53q-hello-2.12.1"
58+
as.NoError(exec.Command("nix", "build", "--no-link", "--refresh", path).Run(), "failed to pull hello path")
59+
60+
// open the sqlite db
61+
db, queries, err := NixV10("file:/nix/var/nix/db/db.sqlite?mode=ro")
62+
as.NoError(err)
63+
defer db.Close()
64+
65+
// query the path we just pulled down
66+
info, err := queries.QueryPathInfo(context.Background(), path)
67+
as.NoError(err)
68+
as.Equal("sha256:f8340af15f7996faded748bea9e2d0b82a6f7c96417b03f7fa8e1a6a873748e8", info.Hash)
69+
as.Equal("/nix/store/qnavcbp5ydyd12asgz7rpr7is7hlswaz-hello-2.12.1.drv", info.Deriver.String)
70+
as.Equal(int64(226560), info.Narsize.Int64)
71+
}

0 commit comments

Comments
 (0)