From 1fb5ec8bf217fabba272c31f13040ab181f406d5 Mon Sep 17 00:00:00 2001 From: Firstyear Date: Sun, 21 Feb 2021 15:04:58 +1000 Subject: [PATCH] vacuum (#365) Fixes #362 moves vacuum to a dedicated task. This is needed as previous vacuuming on startup on large databases could cause the server to fail to start. By making this a task it avoids this error case, and makes the vacuum more predictable, and only run when required. --- kanidm_book/src/administrivia.md | 14 +++++ kanidm_book/src/installing_the_server.md | 11 ++-- kanidmd/src/lib/be/idl_arc_sqlite.rs | 3 +- kanidmd/src/lib/be/idl_sqlite.rs | 65 ++++++++++++++++++++++-- kanidmd/src/lib/be/mod.rs | 5 +- kanidmd/src/lib/core/mod.rs | 35 +++++++++++++ kanidmd/src/lib/macros.rs | 6 +-- kanidmd/src/lib/plugins/macros.rs | 2 +- kanidmd/src/server/main.rs | 46 ++++------------- kanidmd/src/server/opt.rs | 3 ++ 10 files changed, 137 insertions(+), 53 deletions(-) diff --git a/kanidm_book/src/administrivia.md b/kanidm_book/src/administrivia.md index 968a7402c..f0047cbe4 100644 --- a/kanidm_book/src/administrivia.md +++ b/kanidm_book/src/administrivia.md @@ -94,6 +94,20 @@ definitions (this works even though the schema is in the same database!) Generally, reindexing is a rare action and should not normally be required. +# Vacuum + +Vacuuming is the process of reclaiming un-used pages from the sqlite freelists, as well as performing +some data reordering tasks that may make some queries more efficient . It is recommended that you +vacuum after a reindex is performed or when you wish to reclaim space in the database file. + +Vacuum is also able to change the pagesize of the database. After changing db\_fs\_type (which affects +pagesize) in server.toml, you must run a vacuum for this to take effect. + + docker stop + docker run --rm -i -t -v kanidmd:/data \ + kanidm/server:latest /sbin/kanidmd vacuum -c /data/server.toml + docker start + # Verification The server ships with a number of verification utilities to ensure that data is consistent such diff --git a/kanidm_book/src/installing_the_server.md b/kanidm_book/src/installing_the_server.md index 18db12306..95eccc547 100644 --- a/kanidm_book/src/installing_the_server.md +++ b/kanidm_book/src/installing_the_server.md @@ -87,11 +87,12 @@ You will also need a config file in the volume named `server.toml` (Within the c db_path = "/data/kanidm.db" # If you have a known filesystem, kanidm can tune sqlite to match. Valid choices are: # [zfs, other] - # If you are unsure about this leave it as the default (other). - # zfs: - # * sets sqlite pagesize to 64k. You must set recordsize=64k on the zfs filesystem. - # other: - # * sets sqlite pagesize to 4k, matching most filesystems block sizes. + # If you are unsure about this leave it as the default (other). After changing this + # value you must run a vacuum task. + # - zfs: + # * sets sqlite pagesize to 64k. You must set recordsize=64k on the zfs filesystem. + # - other: + # * sets sqlite pagesize to 4k, matching most filesystems block sizes. # db_fs_type = "zfs" # TLS chain and key in pem format. Both must be commented, or both must be present # tls_chain = "/data/chain.pem" diff --git a/kanidmd/src/lib/be/idl_arc_sqlite.rs b/kanidmd/src/lib/be/idl_arc_sqlite.rs index 82a3677aa..33b302c2c 100644 --- a/kanidmd/src/lib/be/idl_arc_sqlite.rs +++ b/kanidmd/src/lib/be/idl_arc_sqlite.rs @@ -846,8 +846,9 @@ impl IdlArcSqlite { path: &str, pool_size: u32, fstype: FsType, + vacuum: bool, ) -> Result { - let db = IdlSqlite::new(audit, path, pool_size, fstype)?; + let db = IdlSqlite::new(audit, path, pool_size, fstype, vacuum)?; let entry_cache = ARCache::new( DEFAULT_CACHE_TARGET, pool_size as usize, diff --git a/kanidmd/src/lib/be/idl_sqlite.rs b/kanidmd/src/lib/be/idl_sqlite.rs index bb4a3c89f..548879a23 100644 --- a/kanidmd/src/lib/be/idl_sqlite.rs +++ b/kanidmd/src/lib/be/idl_sqlite.rs @@ -6,6 +6,7 @@ use idlset::IDLBitRange; use kanidm_proto::v1::{ConsistencyError, OperationError}; use r2d2::Pool; use r2d2_sqlite::SqliteConnectionManager; +use rusqlite::Connection; use rusqlite::OpenFlags; use rusqlite::OptionalExtension; use rusqlite::NO_PARAMS; @@ -1237,6 +1238,7 @@ impl IdlSqlite { path: &str, pool_size: u32, fstype: FsType, + vacuum: bool, ) -> Result { if path == "" { debug_assert!(pool_size == 1); @@ -1249,19 +1251,74 @@ impl IdlSqlite { // Open with multi thread flags and locking options. flags.insert(OpenFlags::SQLITE_OPEN_NO_MUTEX); - // TODO: This probably only needs to be run on first run OR we need a flag - // or something else. Maybe on reindex only? + // We need to run vacuum in the setup else we hit sqlite lock conditions. + if vacuum { + limmediate_warning!( + audit, + "NOTICE: A db vacuum has been requested. This may take a long time ...\n" + ); + + let vconn = Connection::open_with_flags(path, flags).map_err(|e| { + ladmin_error!(audit, "rusqlite error {:?}", e); + OperationError::SQLiteError + })?; + + vconn + .pragma_update(None, "journal_mode", &"DELETE") + .map_err(|e| { + ladmin_error!(audit, "rusqlite journal_mode update error {:?}", e); + OperationError::SQLiteError + })?; + + vconn.close().map_err(|e| { + ladmin_error!(audit, "rusqlite db close error {:?}", e); + OperationError::SQLiteError + })?; + + let vconn = Connection::open_with_flags(path, flags).map_err(|e| { + ladmin_error!(audit, "rusqlite error {:?}", e); + OperationError::SQLiteError + })?; + + vconn + .pragma_update(None, "page_size", &(fstype as u32)) + .map_err(|e| { + ladmin_error!(audit, "rusqlite page_size update error {:?}", e); + OperationError::SQLiteError + })?; + + vconn.execute_batch("VACUUM").map_err(|e| { + ladmin_error!(audit, "rusqlite vacuum error {:?}", e); + OperationError::SQLiteError + })?; + + vconn + .pragma_update(None, "journal_mode", &"WAL") + .map_err(|e| { + ladmin_error!(audit, "rusqlite journal_mode update error {:?}", e); + OperationError::SQLiteError + })?; + + vconn.close().map_err(|e| { + ladmin_error!(audit, "rusqlite db close error {:?}", e); + OperationError::SQLiteError + })?; + + limmediate_warning!(audit, "NOTICE: db vacuum complete\n"); + }; + let manager = SqliteConnectionManager::file(path) .with_init(move |c| { c.execute_batch( format!( - "PRAGMA page_size={}; VACUUM; PRAGMA journal_mode=WAL;", + "PRAGMA page_size={}; PRAGMA journal_mode=WAL;", fstype as u32 ) .as_str(), ) }) .with_flags(flags); + let builder1 = Pool::builder(); let builder2 = builder1.max_size(pool_size); // Look at max_size and thread_pool here for perf later @@ -1303,7 +1360,7 @@ mod tests { #[test] fn test_idl_sqlite_verify() { let mut audit = AuditScope::new("run_test", uuid::Uuid::new_v4(), None); - let be = IdlSqlite::new(&mut audit, "", 1, FsType::Generic).unwrap(); + let be = IdlSqlite::new(&mut audit, "", 1, FsType::Generic, false).unwrap(); let be_w = be.write(); let r = be_w.verify(); assert!(r.len() == 0); diff --git a/kanidmd/src/lib/be/mod.rs b/kanidmd/src/lib/be/mod.rs index 8bf50f3d1..18183c686 100644 --- a/kanidmd/src/lib/be/mod.rs +++ b/kanidmd/src/lib/be/mod.rs @@ -1312,6 +1312,7 @@ impl Backend { mut pool_size: u32, fstype: FsType, idxmeta: Set, + vacuum: bool, ) -> Result { // If in memory, reduce pool to 1 if path == "" { @@ -1322,7 +1323,7 @@ impl Backend { lperf_trace_segment!(audit, "be::new", || { let be = Backend { pool_size: pool_size as usize, - idlayer: Arc::new(IdlArcSqlite::new(audit, path, pool_size, fstype)?), + idlayer: Arc::new(IdlArcSqlite::new(audit, path, pool_size, fstype, vacuum)?), idxmeta: Arc::new(CowCell::new(idxmeta)), }; @@ -1448,7 +1449,7 @@ mod tests { itype: IndexType::EQUALITY, }); - let be = Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta) + let be = Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false) .expect("Failed to setup backend"); let mut be_txn = be.write(); diff --git a/kanidmd/src/lib/core/mod.rs b/kanidmd/src/lib/core/mod.rs index 70c6004fd..21db566ad 100644 --- a/kanidmd/src/lib/core/mod.rs +++ b/kanidmd/src/lib/core/mod.rs @@ -31,6 +31,14 @@ use async_std::task; // === internal setup helpers fn setup_backend(config: &Configuration, schema: &Schema) -> Result { + setup_backend_vacuum(config, schema, false) +} + +fn setup_backend_vacuum( + config: &Configuration, + schema: &Schema, + vacuum: bool, +) -> Result { // Limit the scope of the schema txn. // let schema_txn = task::block_on(schema.write()); let schema_txn = schema.write(); @@ -55,6 +63,7 @@ fn setup_backend(config: &Configuration, schema: &Schema) -> Result s, + Err(e) => { + eprintln!("Failed to setup in memory schema: {:?}", e); + std::process::exit(1); + } + }; + + // The schema doesn't matter here. Vacuum is run as part of db open to avoid + // locking. + let r = setup_backend_vacuum(&config, &schema, true); + + audit.write_log(); + + match r { + Ok(_) => eprintln!("Vacuum Success!"), + Err(e) => { + eprintln!("Vacuum failed: {:?}", e); + std::process::exit(1); + } + }; +} + pub fn domain_rename_core(config: &Configuration, new_domain_name: &str) { let mut audit = AuditScope::new("domain_rename", uuid::Uuid::new_v4(), config.log_level); diff --git a/kanidmd/src/lib/macros.rs b/kanidmd/src/lib/macros.rs index c3d116661..95bca57e6 100644 --- a/kanidmd/src/lib/macros.rs +++ b/kanidmd/src/lib/macros.rs @@ -22,7 +22,7 @@ macro_rules! run_test_no_init { let schema_txn = schema_outer.write_blocking(); schema_txn.reload_idxmeta() }; - let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta) { + let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false) { Ok(be) => be, Err(e) => { audit.write_log(); @@ -66,7 +66,7 @@ macro_rules! run_test { let schema_txn = schema_outer.write_blocking(); schema_txn.reload_idxmeta() }; - let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta) { + let be = match Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false) { Ok(be) => be, Err(e) => { audit.write_log(); @@ -138,7 +138,7 @@ macro_rules! run_idm_test { schema_txn.reload_idxmeta() }; let be = - Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta).expect("Failed to init be"); + Backend::new(&mut audit, "", 1, FsType::Generic, idxmeta, false).expect("Failed to init be"); let test_server = QueryServer::new(be, schema_outer); test_server diff --git a/kanidmd/src/lib/plugins/macros.rs b/kanidmd/src/lib/plugins/macros.rs index 15b98de5e..81be606d8 100644 --- a/kanidmd/src/lib/plugins/macros.rs +++ b/kanidmd/src/lib/plugins/macros.rs @@ -19,7 +19,7 @@ macro_rules! setup_test { let schema_txn = schema_outer.write_blocking(); schema_txn.reload_idxmeta() }; - let be = Backend::new($au, "", 1, FsType::Generic, idxmeta).expect("Failed to init BE"); + let be = Backend::new($au, "", 1, FsType::Generic, idxmeta, false).expect("Failed to init BE"); let qs = QueryServer::new(be, schema_outer); qs.initialise_helper($au, duration_from_epoch_now()) diff --git a/kanidmd/src/server/main.rs b/kanidmd/src/server/main.rs index 6b3b6b64f..82790527f 100644 --- a/kanidmd/src/server/main.rs +++ b/kanidmd/src/server/main.rs @@ -26,7 +26,7 @@ use kanidm::audit::LogLevel; use kanidm::config::Configuration; use kanidm::core::{ backup_server_core, create_server_core, domain_rename_core, recover_account_core, - reindex_server_core, restore_server_core, verify_server_core, + reindex_server_core, restore_server_core, vacuum_server_core, verify_server_core, }; use structopt::StructOpt; @@ -63,9 +63,10 @@ impl ServerConfig { impl KanidmdOpt { fn commonopt(&self) -> &CommonOpt { match self { - KanidmdOpt::Server(sopt) | KanidmdOpt::Verify(sopt) | KanidmdOpt::Reindex(sopt) => { - &sopt - } + KanidmdOpt::Server(sopt) + | KanidmdOpt::Verify(sopt) + | KanidmdOpt::Reindex(sopt) + | KanidmdOpt::Vacuum(sopt) => &sopt, KanidmdOpt::Backup(bopt) => &bopt.commonopts, KanidmdOpt::Restore(ropt) => &ropt.commonopts, KanidmdOpt::RecoverAccount(ropt) => &ropt.commonopts, @@ -223,24 +224,14 @@ async fn main() { match opt { KanidmdOpt::Server(_sopt) => { eprintln!("Running in server mode ..."); - - /* - let mut rt = tokio::runtime::Builder::new() - .threaded_scheduler() - .build() - .unwrap(); - */ - let sctx = create_server_core(config).await; match sctx { Ok(_sctx) => match tokio::signal::ctrl_c().await { Ok(_) => { eprintln!("Ctrl-C received, shutting down"); - // sctx.stop(true).await; } Err(_) => { eprintln!("Invalid signal received, shutting down as a precaution ..."); - // sctx.stop(true).await; } }, Err(_) => { @@ -251,9 +242,6 @@ async fn main() { } KanidmdOpt::Backup(bopt) => { eprintln!("Running in backup mode ..."); - - // config.update_db_path(&bopt.commonopts.db_path); - let p = match bopt.path.to_str() { Some(p) => p, None => { @@ -265,9 +253,6 @@ async fn main() { } KanidmdOpt::Restore(ropt) => { eprintln!("Running in restore mode ..."); - - // config.update_db_path(&ropt.commonopts.db_path); - let p = match ropt.path.to_str() { Some(p) => p, None => { @@ -279,13 +264,10 @@ async fn main() { } KanidmdOpt::Verify(_vopt) => { eprintln!("Running in db verification mode ..."); - - // config.update_db_path(&vopt.db_path); verify_server_core(&config); } KanidmdOpt::RecoverAccount(raopt) => { eprintln!("Running account recovery ..."); - let password = match rpassword::prompt_password_stderr("new password: ") { Ok(pw) => pw, Err(e) => { @@ -293,28 +275,18 @@ async fn main() { std::process::exit(1); } }; - // config.update_db_path(&raopt.commonopts.db_path); - recover_account_core(&config, &raopt.name, &password); } - /* - KanidmdOpt::ResetServerId(vopt) => { - eprintln!("Resetting server id. THIS WILL BREAK REPLICATION"); - - config.update_db_path(&vopt.db_path); - reset_sid_core(config); - } - */ KanidmdOpt::Reindex(_copt) => { eprintln!("Running in reindex mode ..."); - - // config.update_db_path(&copt.db_path); reindex_server_core(&config); } + KanidmdOpt::Vacuum(_copt) => { + eprintln!("Running in vacuum mode ..."); + vacuum_server_core(&config); + } KanidmdOpt::DomainChange(dopt) => { eprintln!("Running in domain name change mode ... this may take a long time ..."); - - // config.update_db_path(&dopt.commonopts.db_path); domain_rename_core(&config, &dopt.new_domain_name); } } diff --git a/kanidmd/src/server/opt.rs b/kanidmd/src/server/opt.rs index 2af63c07d..d7553800c 100644 --- a/kanidmd/src/server/opt.rs +++ b/kanidmd/src/server/opt.rs @@ -66,6 +66,9 @@ enum KanidmdOpt { #[structopt(name = "reindex")] /// Reindex the database (offline) Reindex(CommonOpt), + #[structopt(name = "vacuum")] + /// Vacuum the database to reclaim space or change db_fs_type/page_size (offline) + Vacuum(CommonOpt), #[structopt(name = "domain_name_change")] /// Change the IDM domain name DomainChange(DomainOpt),