Start experimenting with indexing history

Author: Caleb-T-Owens

Cargo.lock

@@ -68,6 +68,7 @@ gitbutler-forge = { path = "crates/gitbutler-forge" }
 gitbutler-hunk-dependency = { path = "crates/gitbutler-hunk-dependency" }
 but-settings = { path = "crates/but-settings" }
 gitbutler-workspace = { path = "crates/gitbutler-workspace" }
+but-inspection = { path = "crates/but-inspection" }
 but = { path = "crates/but" }
 but-testsupport = { path = "crates/but-testsupport" }
 but-rebase = { path = "crates/but-rebase" }

Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "but-inspection"
+version = "0.0.0"
+edition = "2024"
+authors = ["GitButler <[email protected]>"]
+publish = false
+
+[lib]
+doctest = false
+
+[features]
+# Set when building in test-mode to enable features that help with generating repeatable tests.
+testing = []
+
+[dependencies]
+serde = { workspace = true, features = ["std"] }
+bstr.workspace = true
+tracing.workspace = true
+anyhow = "1.0.98"
+gix = { workspace = true, features = [
+    "dirwalk",
+    "credentials",
+    "parallel",
+    "serde",
+    "status",
+] }
+gitbutler-serde.workspace = true
+gitbutler-error.workspace = true
+gitbutler-command-context.workspace = true
+uuid.workspace = true
+toml.workspace = true
+
+[dev-dependencies]

crates/but-inspection/Cargo.toml

@@ -0,0 +1,153 @@
+//! This is due to be replaced with a real vector database. I just got bored
+//! with trying to get them to work.
+use std::{
+    collections::HashMap,
+    hash::{DefaultHasher, Hash as _, Hasher as _},
+    path::PathBuf,
+};
+
+use anyhow::{Result, bail};
+use gitbutler_command_context::CommandContext;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct Hunk {
+    /// The sha of the commit
+    #[serde(with = "gitbutler_serde::object_id")]
+    pub sha: gix::ObjectId,
+    /// Header
+    pub header: String,
+    pub vector: Vec<f32>,
+}
+
+impl Hunk {
+    // We should only ever have one entry per commit & hunk hearder
+    pub fn key(&self) -> u64 {
+        let mut hasher = DefaultHasher::new();
+        (self.sha, &self.header).hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+/// Used to denote a commit has been processed
+pub struct Commit {
+    /// The sha of the commit
+    #[serde(with = "gitbutler_serde::object_id")]
+    pub sha: gix::ObjectId,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Db {
+    pub hunks: Vec<Hunk>,
+    pub commits: Vec<Commit>,
+}
+
+pub struct DbHandle {
+    path: PathBuf,
+}
+
+// TODO: Replace with real vector database
+impl DbHandle {
+    pub fn new(ctx: &CommandContext) -> Self {
+        Self {
+            path: ctx.project().gb_dir().join("inspection.toml"),
+        }
+    }
+
+    pub fn read(&self) -> Result<Db> {
+        if std::fs::exists(&self.path)? {
+            let content = std::fs::read_to_string(&self.path)?;
+            let content: Db = toml::from_str(&content)?;
+            Ok(content)
+        } else {
+            Ok(Db {
+                hunks: vec![],
+                commits: vec![],
+            })
+        }
+    }
+
+    fn write(&self, db: &Db) -> Result<()> {
+        let content = toml::to_string(db)?;
+        std::fs::write(&self.path, content)?;
+        Ok(())
+    }
+
+    pub fn upsert_many_hunks(&self, entries: &[Hunk]) -> Result<Vec<Hunk>> {
+        let mut db = self.read()?;
+        let mut map = db
+            .hunks
+            .into_iter()
+            .map(|e| (e.key(), e))
+            .collect::<HashMap<u64, Hunk>>();
+
+        for e in entries {
+            map.insert(e.key(), e.clone());
+        }
+
+        db.hunks = map.into_values().collect::<Vec<_>>();
+
+        self.upsert_many_commits(
+            &entries
+                .iter()
+                .map(|h| Commit { sha: h.sha })
+                .collect::<Vec<Commit>>(),
+        )?;
+
+        self.write(&db)?;
+
+        Ok(db.hunks)
+    }
+
+    pub fn upsert_many_commits(&self, entries: &[Commit]) -> Result<Vec<Commit>> {
+        let mut db = self.read()?;
+        let mut map = db
+            .commits
+            .into_iter()
+            .map(|e| (e.sha, e))
+            .collect::<HashMap<gix::ObjectId, Commit>>();
+
+        for e in entries {
+            map.insert(e.sha, e.clone());
+        }
+
+        db.commits = map.into_values().collect::<Vec<_>>();
+
+        self.write(&db)?;
+
+        Ok(db.commits)
+    }
+
+    // TODO: Replace with real vector db search rather than a manual implementation.
+    pub fn search_hunks(&self, term: Vec<f32>, cutoff: Option<usize>) -> Result<Vec<(Hunk, f32)>> {
+        let db = self.read()?;
+
+        let mut with_distance = db
+            .hunks
+            .into_iter()
+            .map(|i| {
+                let distance = cosine_distance(&i.vector, &term)?;
+                Ok((i, distance))
+            })
+            .collect::<Result<Vec<(Hunk, f32)>>>()?;
+
+        // Sort decending
+        with_distance.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
+
+        if let Some(cutoff) = cutoff {
+            Ok(with_distance.into_iter().take(cutoff).collect())
+        } else {
+            Ok(with_distance)
+        }
+    }
+}
+
+fn cosine_distance(a: &[f32], b: &[f32]) -> Result<f32> {
+    if a.len() != b.len() {
+        bail!("Vectors MUST be the same length!")
+    };
+
+    let dot: f32 = a.iter().zip(b).map(|(a, b)| a * b).sum();
+    Ok(dot / a.len() as f32)
+}

crates/but-inspection/src/db.rs

@@ -0,0 +1,72 @@
+//! Hello there friend!
+
+use anyhow::Result;
+use gitbutler_command_context::CommandContext;
+use serde::Serialize;
+
+use crate::db::DbHandle;
+
+mod db;
+
+#[derive(Serialize, Debug)]
+pub struct RepositoryIndexStats {
+    /// How many commits we have indexed
+    commits_indexed: usize,
+    /// How many commits are reachable
+    total_commits: usize,
+    /// How many reachable commits are indexed
+    reachable_commits_indexed: usize,
+}
+pub async fn index_stats(ctx: &CommandContext) -> Result<RepositoryIndexStats> {
+    let db_handle = DbHandle::new(ctx);
+    let db = db_handle.read()?;
+    let all_commits = commits_to_index(ctx)?;
+
+    let reachable_commits_indexed = db
+        .commits
+        .iter()
+        .filter(|c| all_commits.contains(&c.sha))
+        .count();
+
+    Ok(RepositoryIndexStats {
+        commits_indexed: db.commits.len(),
+        total_commits: all_commits.len(),
+        reachable_commits_indexed,
+    })
+}
+
+#[derive(Serialize, Debug)]
+pub struct EmbeddingsResult {}
+pub async fn generate_embeddings() -> Result<EmbeddingsResult> {
+    todo!()
+}
+
+/// Lists all commits referenced by references
+fn commits_to_index(ctx: &CommandContext) -> Result<Vec<gix::ObjectId>> {
+    let repo = ctx.gix_repo()?;
+
+    let all_references = repo
+        .references()?
+        .all()?
+        .map(|r| {
+            let id = r.map_err(|_| anyhow::anyhow!("Ahh"))?.try_id();
+            if let Some(id) = id {
+                if id.object()?.kind == gix::object::Kind::Commit {
+                    Ok(Some(id.detach()))
+                } else {
+                    Ok(None)
+                }
+            } else {
+                Ok(None)
+            }
+        })
+        .filter_map(Result::transpose)
+        .collect::<Result<Vec<gix::ObjectId>>>()?;
+
+    let commits = repo.rev_walk(all_references).all()?;
+
+    commits
+        .into_iter()
+        .map(|c| Ok(c?.id))
+        .collect::<Result<Vec<_>>>()
+}

crates/but-inspection/src/lib.rs

@@ -29,5 +29,6 @@ but-action.workspace = true
 but-graph.workspace = true
 but-workspace.workspace = true
 but-settings.workspace = true
+but-inspection.workspace = true
 gitbutler-command-context.workspace = true
 serde_json = "1.0.140"

crates/but/Cargo.toml

@@ -37,4 +37,18 @@ pub enum Subcommands {
         #[clap(long, short = 's', default_value_t = 10)]
         page_size: i64,
     },
+    /// The family of but inspection commands
+    BetaInspect(Inspect),
+}
+
+#[derive(Debug, clap::Parser)]
+pub struct Inspect {
+    #[clap(subcommand)]
+    pub cmd: InspectSubcommands,
+}
+
+#[derive(Debug, clap::Subcommand)]
+pub enum InspectSubcommands {
+    /// Get the current index status
+    Status,
 }

crates/but/src/args.rs

@@ -0,0 +1,27 @@
+use std::path::Path;
+
+use super::print;
+use but_settings::AppSettings;
+use gitbutler_command_context::CommandContext;
+use gitbutler_project::Project;
+
+pub fn status(repo_path: &Path, json: bool) -> anyhow::Result<()> {
+    let repo_path = repo_path.to_owned();
+
+    std::thread::spawn(move || {
+        tokio::runtime::Runtime::new().map(|r| r.block_on(status_inner(&repo_path, json)))
+    })
+    .join()
+    .unwrap()??;
+
+    Ok(())
+}
+
+async fn status_inner(repo_path: &Path, json: bool) -> anyhow::Result<()> {
+    let project = Project::from_path(repo_path).expect("Failed to create project from path");
+    let ctx = CommandContext::open(&project, AppSettings::default())?;
+
+    let outcome = but_inspection::index_stats(&ctx).await?;
+
+    print(&outcome, json)
+}

crates/but/src/command/inspect.rs

@@ -7,6 +7,8 @@ use gitbutler_command_context::CommandContext;
 use gitbutler_project::Project;
 use serde::Serialize;
 
+pub(crate) mod inspect;
+
 pub(crate) fn handle_changes(
     repo_path: &Path,
     json: bool,
@@ -40,7 +42,7 @@ pub(crate) fn list_actions(
     print(&response, json)
 }
 
-fn print<T>(this: &T, json: bool) -> anyhow::Result<()>
+pub(crate) fn print<T>(this: &T, json: bool) -> anyhow::Result<()>
 where
     T: ?Sized + Serialize + std::fmt::Debug,
 {

crates/but/src/command/mod.rs

@@ -2,6 +2,8 @@ use anyhow::Result;
 
 mod args;
 use args::Args;
+
+use crate::args::Inspect;
 mod command;
 mod mcp;
 mod mcp_internal;
@@ -20,5 +22,10 @@ async fn main() -> Result<()> {
         args::Subcommands::ListActions { page, page_size } => {
             command::list_actions(&args.current_dir, args.json, *page, *page_size)
         }
+        args::Subcommands::BetaInspect(Inspect { cmd }) => match cmd {
+            args::InspectSubcommands::Status => {
+                command::inspect::status(&args.current_dir, args.json)
+            }
+        },
     }
 }