Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions src/kvstorage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,13 @@ pub(crate) trait KVStorageTrait {
/// Get total logical bytes (sum of logical_size * refcount for all blobs - what storage would be without dedup)
async fn get_total_logical_bytes(&mut self, bucket: &str) -> Result<i64>;

// Version tracking
/// Get the stored instance version for a bucket
async fn get_version(&mut self, bucket: &str) -> Result<Option<String>>;

/// Store the instance version for a bucket
async fn set_version(&mut self, bucket: &str, version: &str) -> Result<()>;

/// Get deduplicated bytes saved (sum of (refcount - 1) * logical_size for all blobs)
async fn get_deduplicated_bytes_saved(&mut self, bucket: &str) -> Result<i64>;

Expand Down Expand Up @@ -520,4 +527,18 @@ impl KVStorage {
KVStorage::SQLite(storage) => storage.get_pool_stats(),
}
}

pub async fn get_version(&mut self, bucket: &str) -> Result<Option<String>> {
match self {
KVStorage::Postgres(storage) => storage.get_version(bucket).await,
KVStorage::SQLite(storage) => storage.get_version(bucket).await,
}
}

pub async fn set_version(&mut self, bucket: &str, version: &str) -> Result<()> {
match self {
KVStorage::Postgres(storage) => storage.set_version(bucket, version).await,
KVStorage::SQLite(storage) => storage.set_version(bucket, version).await,
}
}
}
30 changes: 30 additions & 0 deletions src/kvstorage/postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,15 @@ impl KVStorageTrait for Postgres {
.execute(&self.pool)
.await?;

sqlx::query(
"CREATE TABLE IF NOT EXISTS version (
bucket VARCHAR(255) NOT NULL PRIMARY KEY,
version VARCHAR(255) NOT NULL
)",
)
.execute(&self.pool)
.await?;

Ok(())
}

Expand Down Expand Up @@ -540,4 +549,25 @@ impl KVStorageTrait for Postgres {
let active_connections = total_connections.saturating_sub(idle_connections);
(active_connections, idle_connections)
}

async fn get_version(&mut self, bucket: &str) -> Result<Option<String>> {
let result: Option<(String,)> =
sqlx::query_as("SELECT version FROM version WHERE bucket = $1")
.bind(bucket)
.fetch_optional(&self.pool)
.await?;
Ok(result.map(|r| r.0))
}

async fn set_version(&mut self, bucket: &str, version: &str) -> Result<()> {
sqlx::query(
"INSERT INTO version (bucket, version) VALUES ($1, $2) \
ON CONFLICT (bucket) DO UPDATE SET version = EXCLUDED.version",
)
.bind(bucket)
.bind(version)
.execute(&self.pool)
.await?;
Ok(())
}
}
22 changes: 22 additions & 0 deletions src/kvstorage/sqlite.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ impl KVStorageTrait for SQLite {
logical_size INTEGER NOT NULL,
compressed_size INTEGER,
PRIMARY KEY (bucket, hash)
);
CREATE TABLE IF NOT EXISTS version (
bucket TEXT NOT NULL PRIMARY KEY,
version TEXT NOT NULL
);",
)
.execute(&self.pool)
Expand Down Expand Up @@ -442,4 +446,22 @@ impl KVStorageTrait for SQLite {
let active_connections = total_connections.saturating_sub(idle_connections);
(active_connections, idle_connections)
}

async fn get_version(&mut self, bucket: &str) -> Result<Option<String>> {
let result: Option<(String,)> =
sqlx::query_as("SELECT version FROM version WHERE bucket = ?1")
.bind(bucket)
.fetch_optional(&self.pool)
.await?;
Ok(result.map(|r| r.0))
}

async fn set_version(&mut self, bucket: &str, version: &str) -> Result<()> {
sqlx::query("INSERT OR REPLACE INTO version (bucket, version) VALUES (?1, ?2)")
.bind(bucket)
.bind(version)
.execute(&self.pool)
.await?;
Ok(())
}
}
13 changes: 13 additions & 0 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use clap::{Parser, Subcommand};
use s3dedup::AppState;
use s3dedup::cleaner::Cleaner;
use s3dedup::config;
use s3dedup::metrics;
use s3dedup::routes::ft::delete_file::ft_delete_file;
use s3dedup::routes::ft::get_file::ft_get_file;
use s3dedup::routes::ft::list_files::ft_list_files;
Expand Down Expand Up @@ -180,6 +181,18 @@ async fn run_s3dedup_server(config_path: Option<&str>, use_env: bool) -> anyhow:
.await
.context("Failed to setup KV storage")?;

// Store instance version
app_state
.kvstorage
.lock()
.await
.set_version(&config.bucket.name, env!("CARGO_PKG_VERSION"))
.await
.context("Failed to store instance version")?;
metrics::INSTANCE_VERSION
.with_label_values(&[env!("CARGO_PKG_VERSION")])
.set(1);

start_background_tasks(app_state.clone(), &config.bucket);

let app = create_router(app_state);
Expand Down
13 changes: 11 additions & 2 deletions src/metrics.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use lazy_static::lazy_static;
use prometheus::{
Encoder, Gauge, HistogramVec, IntCounterVec, IntGauge, TextEncoder, register_gauge,
register_histogram_vec, register_int_counter_vec, register_int_gauge,
Encoder, Gauge, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, TextEncoder,
register_gauge, register_histogram_vec, register_int_counter_vec, register_int_gauge,
register_int_gauge_vec,
};
use serde_json::{Value, json};

Expand Down Expand Up @@ -186,6 +187,14 @@ lazy_static! {
&["storage_type", "reason"]
)
.unwrap();

// Version tracking
pub static ref INSTANCE_VERSION: IntGaugeVec = register_int_gauge_vec!(
"s3dedup_instance_version_info",
"Instance version information (value is always 1, use label 'version' for version string)",
&["version"]
)
.unwrap();
}

#[derive(Clone)]
Expand Down
Loading