WIP: Server Push

backend/hmnet/syncing/discovery.go

@@ -4,6 +4,7 @@
 	"context"
 	"fmt"
 	"seed/backend/blob"
+	"seed/backend/core"
 	docspb "seed/backend/genproto/documents/v3alpha"
 	"seed/backend/hmnet/netutil"
 	"seed/backend/hmnet/syncing/rbsr"
@@ -202,6 +203,8 @@
 }
 
 func loadRBSRStore(conn *sqlite.Conn, dkeys map[discoveryKey]struct{}, store rbsr.Store) error {
+	// List of data to sync here https://seedteamtalks.hyper.media/discussions/things-to-sync-when-pushing-to-a-server?v=bafy2bzacebddt2wpn4vxfqc7zxqvxbq32tyjne23eirpn62vvqo2ce72mjf3g&l
+
 	if err := ensureTempTable(conn, "rbsr_iris"); err != nil {
 		return err
 	}
@@ -210,117 +213,72 @@
 		return err
 	}
 
-	// Fill IRIs.
-	for dkey := range dkeys {
-		if err := sqlitex.Exec(conn, `INSERT OR IGNORE INTO rbsr_iris
-			SELECT id FROM resources WHERE iri = :iri;`, nil, string(dkey.IRI)); err != nil {
-			return err
-		}
-
-		if dkey.Recursive {
-			if err := sqlitex.Exec(conn, `INSERT OR IGNORE INTO rbsr_iris
-				SELECT id FROM resources WHERE iri GLOB :pattern`, nil, string(dkey.IRI)+"/*"); err != nil {
-				return err
-			}
-		}
-
-		// TODO(burdiyan): currently in our database we don't treat comments and other snapshot resources as resources.
-		// Instead comments belong to the document they target, which is different from how we think about them now —
-		// we now think about them as their own state-based resources.
-		// So here we implement a bit of a naughty workaround, to include the blobs into the syncing dataset
-		// if the requested path looks like a TSID of a state-based resource.
-		// We should refactor our database to treat comments as resources and remove this workaround in the future.
-		{
-			space, path, err := dkey.IRI.SpacePath()
-			if err != nil {
-				return err
-			}
-			pathLen := len(path)
-			if pathLen == blob.MinTSIDLength || pathLen == blob.MaxTSIDLength {
-				tsidMaybe := path[1:] // Remove the leading slash from the path.
-				_, _, err := blob.TSID(tsidMaybe).Parse()
-				// Run this query if the path parses like a TSID.
-				// We don't care about the error because it's a best-effort scenario.
-				if err == nil {
-					const q = `INSERT OR IGNORE INTO rbsr_blobs
-						SELECT id
-						FROM structural_blobs
-						WHERE extra_attrs->>'tsid' = :tsid
-						AND author = (SELECT id FROM public_keys WHERE principal = :principal);`
-					if err := sqlitex.Exec(conn, q, nil, tsidMaybe, []byte(space)); err != nil {
-						return err
-					}
-				}
-			}
-		}
-	}
-
-	// Follow all the redirect targets recursively.
-	{
-		const q = `WITH RECURSIVE t (id) AS (
-		    SELECT * FROM rbsr_iris
-		    UNION
-		    SELECT resources.id
-		    FROM structural_blobs sb, resources, t
-		    WHERE (t.id = sb.resource AND sb.type = 'Ref')
-		    AND sb.extra_attrs->>'redirect' IS NOT NULL
-		    AND sb.extra_attrs->>'redirect' = resources.iri
-		)
-		SELECT * FROM t;`
-
-		// TODO(burdiyan): this query doesn't do anything, I forget why it's here.
+	if err := fillTables(conn, dkeys); err != nil {
+		return err
 	}
 
-	// Fill Refs.
+	var linkIRIs map[discoveryKey]struct{} = make(map[discoveryKey]struct{})
+	// Fill Links.
 	{
-		const q = `INSERT OR IGNORE INTO rbsr_blobs
-			SELECT sb.id
-			FROM structural_blobs sb
-			LEFT OUTER JOIN stashed_blobs ON stashed_blobs.id = sb.id
-			WHERE resource IN rbsr_iris
-			AND type = 'Ref'`
-
-		if err := sqlitex.Exec(conn, q, nil); err != nil {
+		const q = `
+			WITH genesis (id) AS (
+				SELECT distinct genesis_blob FROM resources WHERE id IN rbsr_iris
+			), linked_changes (id) AS (
+				SELECT id FROM structural_blobs WHERE genesis_blob IN (SELECT id FROM genesis)
+				UNION ALL
+				SELECT id from genesis
+			)
+			SELECT r.iri,
+			rl.is_pinned,
+			rl.extra_attrs->>'v' AS version
+			FROM resources r
+			JOIN resource_links rl ON r.id = rl.target
+			WHERE rl.source IN linked_changes
+			GROUP BY r.iri, version, rl.is_pinned;`
+
+		if err := sqlitex.Exec(conn, q, func(stmt *sqlite.Stmt) error {
+			var iri = blob.IRI(stmt.ColumnText(0))
+			var version = blob.Version(stmt.ColumnText(2))
+			var isPinned = stmt.ColumnInt(1) != 0
+			dKey := discoveryKey{IRI: iri, Version: "", Recursive: false}
+			if isPinned && version != "" {
+				// If it's pinned, we want to make sure we get the specific version.
+				dKey = discoveryKey{IRI: iri, Version: version, Recursive: false}
+			}
+			linkIRIs[dKey] = struct{}{}
+			return nil
+		}); err != nil {
 			return err
 		}
 	}
-
-	// Fill Changes based on Refs.
+	// Fill Citations.
 	{
-		const q = `WITH RECURSIVE
-			changes (id) AS (
-			    SELECT target
-			    FROM blob_links bl
-			    JOIN rbsr_blobs rb ON rb.id = bl.source
-			        AND bl.type = 'ref/head'
-			    UNION
-			    SELECT target
-			    FROM blob_links bl
-			    JOIN changes c ON c.id = bl.source
-			        AND bl.type = 'change/dep'
+		if err := sqlitex.ExecTransient(conn, listCitations(), func(stmt *sqlite.Stmt) error {
+			var (
+				author    = core.Principal(stmt.ColumnBytesUnsafe(0)).String()
+				tsid      = blob.TSID(stmt.ColumnText(1))
+				isDeleted = stmt.ColumnText(2) == "1"
+				source    = stmt.ColumnText(3)
+				blobType  = stmt.ColumnText(4)
 			)
-			INSERT OR IGNORE INTO rbsr_blobs
-			SELECT id FROM changes;`
-
-		if err := sqlitex.Exec(conn, q, nil); err != nil {
-			return err
-		}
-	}
 
-	// Fill Capabilities and the rest of the related blob types.
-	{
-		const q = `INSERT OR IGNORE INTO rbsr_blobs
-			SELECT sb.id
-			FROM structural_blobs sb
-			LEFT OUTER JOIN stashed_blobs ON stashed_blobs.id = sb.id
-			WHERE resource IN rbsr_iris
-			AND sb.type IN ('Capability', 'Comment', 'Profile', 'Contact')`
+			if blobType == "Comment" {
+				source = "hm://" + author + "/" + tsid.String()
 
-		if err := sqlitex.Exec(conn, q, nil); err != nil {
+			}
+			if isDeleted {
+				return nil
+			}
+			dKey := discoveryKey{IRI: blob.IRI(source)}
+			linkIRIs[dKey] = struct{}{}
+			return nil
+		}); err != nil {
 			return err
 		}
 	}
-
+	if err := fillTables(conn, linkIRIs); err != nil {
+		return err
+	}
 	// Find recursively all the agent capabilities for authors of the blobs we've currently selected,
 	// until we can't find any more.
 	for {
@@ -388,6 +346,120 @@
 	return nil
 }
 
+func fillTables(conn *sqlite.Conn, dkeys map[discoveryKey]struct{}) error {
+	// Fill IRIs.
+	for dkey := range dkeys {
+		if err := sqlitex.Exec(conn, `INSERT OR IGNORE INTO rbsr_iris
+				SELECT id FROM resources WHERE iri = :iri;`, nil, string(dkey.IRI)); err != nil {
+			return err
+		}
+
+		if dkey.Recursive {
+			if err := sqlitex.Exec(conn, `INSERT OR IGNORE INTO rbsr_iris
+					SELECT id FROM resources WHERE iri GLOB :pattern`, nil, string(dkey.IRI)+"/*"); err != nil {
+				return err
+			}
+		}
+
+		// TODO(burdiyan): currently in our database we don't treat comments and other snapshot resources as resources.
+		// Instead comments belong to the document they target, which is different from how we think about them now —
+		// we now think about them as their own state-based resources.
+		// So here we implement a bit of a naughty workaround, to include the blobs into the syncing dataset
+		// if the requested path looks like a TSID of a state-based resource.
+		// We should refactor our database to treat comments as resources and remove this workaround in the future.
+		{
+			space, path, err := dkey.IRI.SpacePath()
+			if err != nil {
+				return err
+			}
+			pathLen := len(path)
+			if pathLen == blob.MinTSIDLength || pathLen == blob.MaxTSIDLength {
+				tsidMaybe := path[1:] // Remove the leading slash from the path.
+				_, _, err := blob.TSID(tsidMaybe).Parse()
+				// Run this query if the path parses like a TSID.
+				// We don't care about the error because it's a best-effort scenario.
+				if err == nil {
+					const q = `INSERT OR IGNORE INTO rbsr_blobs
+							SELECT id
+							FROM structural_blobs
+							WHERE extra_attrs->>'tsid' = :tsid
+							AND author = (SELECT id FROM public_keys WHERE principal = :principal);`
+					if err := sqlitex.Exec(conn, q, nil, tsidMaybe, []byte(space)); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	// Follow all the redirect targets recursively.
+	{
+		const q = `WITH RECURSIVE t (id) AS (
+				SELECT * FROM rbsr_iris
+				UNION
+				SELECT resources.id
+				FROM structural_blobs sb, resources, t
+				WHERE (t.id = sb.resource AND sb.type = 'Ref')
+				AND sb.extra_attrs->>'redirect' IS NOT NULL
+				AND sb.extra_attrs->>'redirect' = resources.iri
+			)
+			SELECT * FROM t;`
+
+		// TODO(burdiyan): this query doesn't do anything, I forget why it's here.
+	}
+
+	// Fill Refs.
+	{
+		const q = `INSERT OR IGNORE INTO rbsr_blobs
+				SELECT sb.id
+				FROM structural_blobs sb
+				LEFT OUTER JOIN stashed_blobs ON stashed_blobs.id = sb.id
+				WHERE resource IN rbsr_iris
+				AND type = 'Ref'`
+
+		if err := sqlitex.Exec(conn, q, nil); err != nil {
+			return err
+		}
+	}
+
+	// Fill Changes based on Refs.
+	{
+		const q = `WITH RECURSIVE
+				changes (id) AS (
+					SELECT target
+					FROM blob_links bl
+					JOIN rbsr_blobs rb ON rb.id = bl.source
+						AND bl.type = 'ref/head'
+					UNION
+					SELECT target
+					FROM blob_links bl
+					JOIN changes c ON c.id = bl.source
+						AND bl.type = 'change/dep'
+				)
+				INSERT OR IGNORE INTO rbsr_blobs
+				SELECT id FROM changes;`
+
+		if err := sqlitex.Exec(conn, q, nil); err != nil {
+			return err
+		}
+	}
+
+	// Fill Capabilities and the rest of the related blob types.
+	{
+		const q = `INSERT OR IGNORE INTO rbsr_blobs
+				SELECT sb.id
+				FROM structural_blobs sb
+				LEFT OUTER JOIN stashed_blobs ON stashed_blobs.id = sb.id
+				WHERE resource IN rbsr_iris
+				AND sb.type IN ('Capability', 'Comment', 'Profile', 'Contact')`
+
+		if err := sqlitex.Exec(conn, q, nil); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func ensureTempTable(conn *sqlite.Conn, name string) error {
 	err := sqlitex.Exec(conn, "DELETE FROM "+name, nil)
 	if err == nil {
@@ -404,3 +476,22 @@
 	WHERE iri = :iri
 	LIMIT 1;
 `)
+
+var listCitations = dqb.Str(`
+SELECT distinct
+	public_keys.principal AS main_author,
+	structural_blobs.extra_attrs->>'tsid' AS tsid,
+	structural_blobs.extra_attrs->>'deleted' as is_deleted,
+	r.iri AS source_iri,
+	structural_blobs.type AS blob_type
+FROM resource_links
+JOIN structural_blobs ON structural_blobs.id = resource_links.source
+JOIN blobs INDEXED BY blobs_metadata ON blobs.id = structural_blobs.id
+JOIN public_keys ON public_keys.id = structural_blobs.author
+LEFT JOIN resources r
+  ON r.genesis_blob = CASE
+        WHEN structural_blobs.type != 'Change' THEN structural_blobs.genesis_blob
+        ELSE coalesce(structural_blobs.genesis_blob, structural_blobs.id)
+     END
+WHERE resource_links.target IN rbsr_iris;
+`)

backend/hmnet/syncing/discovery_test.go

@@ -0,0 +1,42 @@
+package syncing
+
+import (
+	"seed/backend/hmnet/syncing/rbsr"
+	"seed/backend/storage"
+	"seed/backend/util/colx"
+	"seed/backend/util/sqlite"
+	"seed/backend/util/sqlite/sqlitex"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadExternalStore(t *testing.T) {
+	t.Skip("This uses external database for local test only")
+	t.Parallel()
+	pool := loadLocalDB(t, "/home/julio/.config/Seed-local/daemon/db/db.sqlite")
+	store := rbsr.NewSliceStore()
+	// Create RBSR store once for reuse across all peers.
+	dKeys := colx.HashSet[discoveryKey]{
+		discoveryKey{
+			IRI: "hm://z6Mkq9emq1yUBq4KSeiSH5yzgNBJSnidPVqFnTpzjCdLxB3R/tests/doc1",
+		}: {},
+	}
+	err := pool.WithSave(t.Context(), func(conn *sqlite.Conn) error {
+		return loadRBSRStore(conn, dKeys, store)
+	})
+	require.NoError(t, err)
+
+}
+
+func loadLocalDB(t testing.TB, path string) *sqlitex.Pool {
+	t.Helper()
+
+	pool, err := storage.OpenSQLite(path, 0, 6)
+
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, pool.Close())
+	})
+	return pool
+}

frontend/apps/web/app/document.tsx

@@ -829,7 +829,7 @@ function InnerDocumentPage(
                 <>
                   <MobileInteractionCardCollapsed
                     onClick={() => {
-                      setDocumentPanel({type: 'activity'})
+                      setDocumentPanel({type: 'discussions'})
                       // setMobilePanelOpen(true)
                     }}
                     commentsCount={interactionSummary.data?.comments || 0}

frontend/packages/ui/src/document-content.tsx

@@ -2575,7 +2575,7 @@ export function InlineEmbedButton({
       onMouseLeave={() => props.onHoverOut?.(entityId)}
       className={cn(
         'font-bold text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300',
-        hasRangeHighlight && 'hm-embed-range bg-brand-10 hover:cursor-default',
+        hasRangeHighlight && 'hm-embed-range hover:cursor-default',
       )}
       data-inline-embed={packHmId(entityId)}
       // this data attribute is used by the hypermedia highlight component