vacuum (#365)

Fixes #362 moves vacuum to a dedicated task. This is needed as previous vacuuming on startup on large databases could cause the server to fail to start. By making this a task it avoids this error case, and makes the vacuum more predictable, and only run when required.

kanidm_book/src/administrivia.md

@@ -94,6 +94,20 @@ definitions (this works even though the schema is in the same database!)
 
 Generally, reindexing is a rare action and should not normally be required.
 
+# Vacuum
+
+Vacuuming is the process of reclaiming un-used pages from the sqlite freelists, as well as performing
+some data reordering tasks that may make some queries more efficient . It is recommended that you
+vacuum after a reindex is performed or when you wish to reclaim space in the database file.
+
+Vacuum is also able to change the pagesize of the database. After changing db\_fs\_type (which affects 
+pagesize) in server.toml, you must run a vacuum for this to take effect.
+
+    docker stop <container name>
+    docker run --rm -i -t -v kanidmd:/data \
+        kanidm/server:latest /sbin/kanidmd vacuum -c /data/server.toml
+    docker start <container name>
+
 # Verification
 
 The server ships with a number of verification utilities to ensure that data is consistent such

kanidm_book/src/installing_the_server.md

@@ -87,11 +87,12 @@ You will also need a config file in the volume named `server.toml` (Within the c
     db_path = "/data/kanidm.db"
     #   If you have a known filesystem, kanidm can tune sqlite to match. Valid choices are:
     #   [zfs, other]
-    #   If you are unsure about this leave it as the default (other).
+    #   If you are unsure about this leave it as the default (other). After changing this
-    #   zfs:
+    #   value you must run a vacuum task.
-    #   * sets sqlite pagesize to 64k. You must set recordsize=64k on the zfs filesystem.
+    #   - zfs:
-    #   other:
+    #     * sets sqlite pagesize to 64k. You must set recordsize=64k on the zfs filesystem.
-    #   * sets sqlite pagesize to 4k, matching most filesystems block sizes.
+    #   - other:
+    #     * sets sqlite pagesize to 4k, matching most filesystems block sizes.
     # db_fs_type = "zfs"
     #   TLS chain and key in pem format. Both must be commented, or both must be present
     # tls_chain = "/data/chain.pem"

kanidmd/src/lib/be/idl_arc_sqlite.rs

@@ -846,8 +846,9 @@ impl IdlArcSqlite {
         path: &str,
         pool_size: u32,
         fstype: FsType,
+        vacuum: bool,
     ) -> Result<Self, OperationError> {
-        let db = IdlSqlite::new(audit, path, pool_size, fstype)?;
+        let db = IdlSqlite::new(audit, path, pool_size, fstype, vacuum)?;
         let entry_cache = ARCache::new(
             DEFAULT_CACHE_TARGET,
             pool_size as usize,

kanidmd/src/lib/be/idl_sqlite.rs

@@ -6,6 +6,7 @@ use idlset::IDLBitRange;
 use kanidm_proto::v1::{ConsistencyError, OperationError};
 use r2d2::Pool;
 use r2d2_sqlite::SqliteConnectionManager;
+use rusqlite::Connection;
 use rusqlite::OpenFlags;
 use rusqlite::OptionalExtension;
 use rusqlite::NO_PARAMS;
@@ -1237,6 +1238,7 @@ impl IdlSqlite {
         path: &str,
         pool_size: u32,
         fstype: FsType,
+        vacuum: bool,
     ) -> Result<Self, OperationError> {
         if path == "" {
             debug_assert!(pool_size == 1);
@@ -1249,19 +1251,74 @@ impl IdlSqlite {
         // Open with multi thread flags and locking options.
         flags.insert(OpenFlags::SQLITE_OPEN_NO_MUTEX);
 
-        // TODO: This probably only needs to be run on first run OR we need a flag
+        // We need to run vacuum in the setup else we hit sqlite lock conditions.
-        // or something else. Maybe on reindex only?
+        if vacuum {
+            limmediate_warning!(
+                audit,
+                "NOTICE: A db vacuum has been requested. This may take a long time ...\n"
+            );
+
+            let vconn = Connection::open_with_flags(path, flags).map_err(|e| {
+                ladmin_error!(audit, "rusqlite error {:?}", e);
+                OperationError::SQLiteError
+            })?;
+
+            vconn
+                .pragma_update(None, "journal_mode", &"DELETE")
+                .map_err(|e| {
+                    ladmin_error!(audit, "rusqlite journal_mode update error {:?}", e);
+                    OperationError::SQLiteError
+                })?;
+
+            vconn.close().map_err(|e| {
+                ladmin_error!(audit, "rusqlite db close error {:?}", e);
+                OperationError::SQLiteError
+            })?;
+
+            let vconn = Connection::open_with_flags(path, flags).map_err(|e| {
+                ladmin_error!(audit, "rusqlite error {:?}", e);
+                OperationError::SQLiteError
+            })?;
+
+            vconn
+                .pragma_update(None, "page_size", &(fstype as u32))
+                .map_err(|e| {
+                    ladmin_error!(audit, "rusqlite page_size update error {:?}", e);
+                    OperationError::SQLiteError
+                })?;
+
+            vconn.execute_batch("VACUUM").map_err(|e| {
+                ladmin_error!(audit, "rusqlite vacuum error {:?}", e);
+                OperationError::SQLiteError
+            })?;
+
+            vconn
+                .pragma_update(None, "journal_mode", &"WAL")
+                .map_err(|e| {
+                    ladmin_error!(audit, "rusqlite journal_mode update error {:?}", e);
+                    OperationError::SQLiteError
+                })?;
+
+            vconn.close().map_err(|e| {
+                ladmin_error!(audit, "rusqlite db close error {:?}", e);
+                OperationError::SQLiteError
+            })?;
+
+            limmediate_warning!(audit, "NOTICE: db vacuum complete\n");
+        };
+
         let manager = SqliteConnectionManager::file(path)
             .with_init(move |c| {
                 c.execute_batch(
                     format!(
-                        "PRAGMA page_size={}; VACUUM; PRAGMA journal_mode=WAL;",
+                        "PRAGMA page_size={}; PRAGMA journal_mode=WAL;",
                         fstype as u32
                     )
                     .as_str(),
                 )
             })
             .with_flags(flags);
+
         let builder1 = Pool::builder();
         let builder2 = builder1.max_size(pool_size);
         // Look at max_size and thread_pool here for perf later
@@ -1303,7 +1360,7 @@ mod tests {
     #[test]
     fn test_idl_sqlite_verify() {
         let mut audit = AuditScope::new("run_test", uuid::Uuid::new_v4(), None);
-        let be = IdlSqlite::new(&mut audit, "", 1, FsType::Generic).unwrap();
+        let be = IdlSqlite::new(&mut audit, "", 1, FsType::Generic, false).unwrap();
         let be_w = be.write();
         let r = be_w.verify();
         assert!(r.len() == 0);

kanidmd/src/lib/be/mod.rs

@@ -1312,6 +1312,7 @@ impl Backend {
         mut pool_size: u32,
         fstype: FsType,
         idxmeta: Set<IdxKey>,
+        vacuum: bool,
     ) -> Result<Self, OperationError> {
         // If in memory, reduce pool to 1
         if path == "" {
@@ -1322,7 +1323,7 @@ impl Backend {
         lperf_trace_segment!(audit, "be::new", || {
             let be = Backend {
                 pool_size: pool_size as usize,
-                idlayer: Arc::new(IdlArcSqlite::new(audit, path, pool_size, fstype)?),
+                idlayer: Arc::new(IdlArcSqlite::new(audit, path, pool_size, fstype, vacuum)?),
                 idxmeta: Arc::new(CowCell::new(idxmeta)),
             };
 
@@ -1448,7 +1449,7 @@ mod tests {
                 itype: IndexType::EQUALITY,
             });
 
-            let be = Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta)
+            let be = Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false)
                 .expect("Failed to setup backend");
 
             let mut be_txn = be.write();

kanidmd/src/lib/core/mod.rs

@@ -31,6 +31,14 @@ use async_std::task;
 // === internal setup helpers
 
 fn setup_backend(config: &Configuration, schema: &Schema) -> Result<Backend, OperationError> {
+    setup_backend_vacuum(config, schema, false)
+}
+
+fn setup_backend_vacuum(
+    config: &Configuration,
+    schema: &Schema,
+    vacuum: bool,
+) -> Result<Backend, OperationError> {
     // Limit the scope of the schema txn.
     // let schema_txn = task::block_on(schema.write());
     let schema_txn = schema.write();
@@ -55,6 +63,7 @@ fn setup_backend(config: &Configuration, schema: &Schema) -> Result<Backend, Ope
         pool_size,
         fstype,
         idxmeta,
+        vacuum,
     );
     // debug!
     audit_be.write_log();
@@ -257,6 +266,32 @@ pub fn reindex_server_core(config: &Configuration) {
     };
 }
 
+pub fn vacuum_server_core(config: &Configuration) {
+    let mut audit = AuditScope::new("server_vacuum", uuid::Uuid::new_v4(), config.log_level);
+
+    let schema = match Schema::new(&mut audit) {
+        Ok(s) => s,
+        Err(e) => {
+            eprintln!("Failed to setup in memory schema: {:?}", e);
+            std::process::exit(1);
+        }
+    };
+
+    // The schema doesn't matter here. Vacuum is run as part of db open to avoid
+    // locking.
+    let r = setup_backend_vacuum(&config, &schema, true);
+
+    audit.write_log();
+
+    match r {
+        Ok(_) => eprintln!("Vacuum Success!"),
+        Err(e) => {
+            eprintln!("Vacuum failed: {:?}", e);
+            std::process::exit(1);
+        }
+    };
+}
+
 pub fn domain_rename_core(config: &Configuration, new_domain_name: &str) {
     let mut audit = AuditScope::new("domain_rename", uuid::Uuid::new_v4(), config.log_level);
 

kanidmd/src/lib/macros.rs

@@ -22,7 +22,7 @@ macro_rules! run_test_no_init {
             let schema_txn = schema_outer.write_blocking();
             schema_txn.reload_idxmeta()
         };
-        let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta) {
+        let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false) {
             Ok(be) => be,
             Err(e) => {
                 audit.write_log();
@@ -66,7 +66,7 @@ macro_rules! run_test {
             let schema_txn = schema_outer.write_blocking();
             schema_txn.reload_idxmeta()
         };
-        let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta) {
+        let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false) {
             Ok(be) => be,
             Err(e) => {
                 audit.write_log();
@@ -138,7 +138,7 @@ macro_rules! run_idm_test {
             schema_txn.reload_idxmeta()
         };
         let be =
-            Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta).expect("Failed to init be");
+            Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false).expect("Failed to init be");
 
         let test_server = QueryServer::new(be, schema_outer);
         test_server

kanidmd/src/lib/plugins/macros.rs

@@ -19,7 +19,7 @@ macro_rules! setup_test {
             let schema_txn = schema_outer.write_blocking();
             schema_txn.reload_idxmeta()
         };
-        let be = Backend::new($au, "", 1, FsType::Generic, idxmeta).expect("Failed to init BE");
+        let be = Backend::new($au, "", 1, FsType::Generic, idxmeta, false).expect("Failed to init BE");
 
         let qs = QueryServer::new(be, schema_outer);
         qs.initialise_helper($au, duration_from_epoch_now())

kanidmd/src/server/main.rs

@@ -26,7 +26,7 @@ use kanidm::audit::LogLevel;
 use kanidm::config::Configuration;
 use kanidm::core::{
     backup_server_core, create_server_core, domain_rename_core, recover_account_core,
-    reindex_server_core, restore_server_core, verify_server_core,
+    reindex_server_core, restore_server_core, vacuum_server_core, verify_server_core,
 };
 
 use structopt::StructOpt;
@@ -63,9 +63,10 @@ impl ServerConfig {
 impl KanidmdOpt {
     fn commonopt(&self) -> &CommonOpt {
         match self {
-            KanidmdOpt::Server(sopt) | KanidmdOpt::Verify(sopt) | KanidmdOpt::Reindex(sopt) => {
+            KanidmdOpt::Server(sopt)
-                &sopt
+            | KanidmdOpt::Verify(sopt)
-            }
+            | KanidmdOpt::Reindex(sopt)
+            | KanidmdOpt::Vacuum(sopt) => &sopt,
             KanidmdOpt::Backup(bopt) => &bopt.commonopts,
             KanidmdOpt::Restore(ropt) => &ropt.commonopts,
             KanidmdOpt::RecoverAccount(ropt) => &ropt.commonopts,
@@ -223,24 +224,14 @@ async fn main() {
     match opt {
         KanidmdOpt::Server(_sopt) => {
             eprintln!("Running in server mode ...");
-
-            /*
-            let mut rt = tokio::runtime::Builder::new()
-                .threaded_scheduler()
-                .build()
-                .unwrap();
-            */
-
             let sctx = create_server_core(config).await;
             match sctx {
                 Ok(_sctx) => match tokio::signal::ctrl_c().await {
                     Ok(_) => {
                         eprintln!("Ctrl-C received, shutting down");
-                        // sctx.stop(true).await;
                     }
                     Err(_) => {
                         eprintln!("Invalid signal received, shutting down as a precaution ...");
-                        // sctx.stop(true).await;
                     }
                 },
                 Err(_) => {
@@ -251,9 +242,6 @@ async fn main() {
         }
         KanidmdOpt::Backup(bopt) => {
             eprintln!("Running in backup mode ...");
-
-            // config.update_db_path(&bopt.commonopts.db_path);
-
             let p = match bopt.path.to_str() {
                 Some(p) => p,
                 None => {
@@ -265,9 +253,6 @@ async fn main() {
         }
         KanidmdOpt::Restore(ropt) => {
             eprintln!("Running in restore mode ...");
-
-            // config.update_db_path(&ropt.commonopts.db_path);
-
             let p = match ropt.path.to_str() {
                 Some(p) => p,
                 None => {
@@ -279,13 +264,10 @@ async fn main() {
         }
         KanidmdOpt::Verify(_vopt) => {
             eprintln!("Running in db verification mode ...");
-
-            // config.update_db_path(&vopt.db_path);
             verify_server_core(&config);
         }
         KanidmdOpt::RecoverAccount(raopt) => {
             eprintln!("Running account recovery ...");
-
             let password = match rpassword::prompt_password_stderr("new password: ") {
                 Ok(pw) => pw,
                 Err(e) => {
@@ -293,28 +275,18 @@ async fn main() {
                     std::process::exit(1);
                 }
             };
-            // config.update_db_path(&raopt.commonopts.db_path);
-
             recover_account_core(&config, &raopt.name, &password);
         }
-        /*
-        KanidmdOpt::ResetServerId(vopt) => {
-            eprintln!("Resetting server id. THIS WILL BREAK REPLICATION");
-
-            config.update_db_path(&vopt.db_path);
-            reset_sid_core(config);
-        }
-        */
         KanidmdOpt::Reindex(_copt) => {
             eprintln!("Running in reindex mode ...");
-
-            // config.update_db_path(&copt.db_path);
             reindex_server_core(&config);
         }
+        KanidmdOpt::Vacuum(_copt) => {
+            eprintln!("Running in vacuum mode ...");
+            vacuum_server_core(&config);
+        }
         KanidmdOpt::DomainChange(dopt) => {
             eprintln!("Running in domain name change mode ... this may take a long time ...");
-
-            // config.update_db_path(&dopt.commonopts.db_path);
             domain_rename_core(&config, &dopt.new_domain_name);
         }
     }

kanidmd/src/server/opt.rs

@@ -66,6 +66,9 @@ enum KanidmdOpt {
     #[structopt(name = "reindex")]
     /// Reindex the database (offline)
     Reindex(CommonOpt),
+    #[structopt(name = "vacuum")]
+    /// Vacuum the database to reclaim space or change db_fs_type/page_size (offline)
+    Vacuum(CommonOpt),
     #[structopt(name = "domain_name_change")]
     /// Change the IDM domain name
     DomainChange(DomainOpt),