mart-w/kanidm
1
0
Fork 0
mirror of https://github.com/kanidm/kanidm.git synced 2025-02-23 20:47:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Start to remove audit scope :) (#574)

Browse source
This commit is contained in:
Firstyear 2021-08-26 11:48:03 +10:00 committed by GitHub
parent 09e83a98c6
commit 1080e5d0b4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 591 additions and 1092 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

779
kanidmd/src/lib/be/idl_arc_sqlite.rs
View file

File diff suppressed because it is too large Load diff

466
kanidmd/src/lib/be/idl_sqlite.rs
View file

@ -1,4 +1,3 @@
use crate::audit::AuditScope;
use crate::be::{BackendConfig, IdList, IdRawEntry, IdxKey, IdxSlope}; use crate::be::{BackendConfig, IdList, IdRawEntry, IdxKey, IdxSlope};
use crate::entry::{Entry, EntryCommitted, EntrySealed}; use crate::entry::{Entry, EntryCommitted, EntrySealed};
use crate::prelude::*; use crate::prelude::*;
@ -116,25 +115,18 @@ pub trait IdlSqliteTransaction {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn get_identry( fn get_identry(
&self, &self,
au: &mut AuditScope,
idl: &IdList, idl: &IdList,
) -> Result<Vec<Entry<EntrySealed, EntryCommitted>>, OperationError> { ) -> Result<Vec<Entry<EntrySealed, EntryCommitted>>, OperationError> {
spanned!("be::idl_sqlite::get_identry", { spanned!("be::idl_sqlite::get_identry", {
lperf_trace_segment!(au, "be::idl_sqlite::get_identry", || { self.get_identry_raw(idl)?
self.get_identry_raw(au, idl)? .into_iter()
.into_iter() .map(|ide| ide.into_entry())
.map(|ide| ide.into_entry(au)) .collect()
.collect()
})
}) })
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn get_identry_raw( fn get_identry_raw(&self, idl: &IdList) -> Result<Vec<IdRawEntry>, OperationError> {
&self,
_au: &mut AuditScope,
idl: &IdList,
) -> Result<Vec<IdRawEntry>, OperationError> {
// is the idl allids? // is the idl allids?
match idl { match idl {
IdList::AllIds => { IdList::AllIds => {
@ -204,7 +196,7 @@ pub trait IdlSqliteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn exists_table(&self, _audit: &mut AuditScope, tname: &str) -> Result<bool, OperationError> { fn exists_table(&self, tname: &str) -> Result<bool, OperationError> {
let mut stmt = self let mut stmt = self
.get_conn() .get_conn()
.prepare("SELECT COUNT(name) from sqlite_master where name = :tname") .prepare("SELECT COUNT(name) from sqlite_master where name = :tname")
@ -220,153 +212,122 @@ pub trait IdlSqliteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn exists_idx( fn exists_idx(&self, attr: &str, itype: &IndexType) -> Result<bool, OperationError> {
&self,
audit: &mut AuditScope,
attr: &str,
itype: &IndexType,
) -> Result<bool, OperationError> {
let tname = format!("idx_{}_{}", itype.as_idx_str(), attr); let tname = format!("idx_{}_{}", itype.as_idx_str(), attr);
self.exists_table(audit, &tname) self.exists_table(&tname)
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn get_idl( fn get_idl(
&self, &self,
audit: &mut AuditScope,
attr: &str, attr: &str,
itype: &IndexType, itype: &IndexType,
idx_key: &str, idx_key: &str,
) -> Result<Option<IDLBitRange>, OperationError> { ) -> Result<Option<IDLBitRange>, OperationError> {
spanned!("be::idl_sqlite::get_idl", { spanned!("be::idl_sqlite::get_idl", {
lperf_trace_segment!(audit, "be::idl_sqlite::get_idl", || { if !(self.exists_idx(attr, itype)?) {
if !(self.exists_idx(audit, attr, itype)?) { filter_error!("Index {:?} {:?} not found", itype, attr);
filter_error!("Index {:?} {:?} not found", itype, attr); return Ok(None);
lfilter_error!(audit, "Index {:?} {:?} not found", itype, attr); }
return Ok(None); // The table exists - lets now get the actual index itself.
}
// The table exists - lets now get the actual index itself.
let query = format!( let query = format!(
"SELECT idl FROM idx_{}_{} WHERE key = :idx_key", "SELECT idl FROM idx_{}_{} WHERE key = :idx_key",
itype.as_idx_str(), itype.as_idx_str(),
attr attr
); );
let mut stmt = self let mut stmt = self
.get_conn() .get_conn()
.prepare(query.as_str()) .prepare(query.as_str())
.map_err(sqlite_error)?; .map_err(sqlite_error)?;
let idl_raw: Option<Vec<u8>> = stmt let idl_raw: Option<Vec<u8>> = stmt
.query_row(&[(":idx_key", &idx_key)], |row| row.get(0)) .query_row(&[(":idx_key", &idx_key)], |row| row.get(0))
// We don't mind if it doesn't exist // We don't mind if it doesn't exist
.optional() .optional()
.map_err(sqlite_error)?; .map_err(sqlite_error)?;
let idl = match idl_raw { let idl = match idl_raw {
Some(d) => serde_cbor::from_slice(d.as_slice()).map_err(serde_cbor_error)?, Some(d) => serde_cbor::from_slice(d.as_slice()).map_err(serde_cbor_error)?,
// We don't have this value, it must be empty (or we // We don't have this value, it must be empty (or we
// have a corrupted index ..... // have a corrupted index .....
None => IDLBitRange::new(), None => IDLBitRange::new(),
}; };
trace!(%idl, "Got idl for index {:?} {:?}", itype, attr); trace!(%idl, "Got idl for index {:?} {:?}", itype, attr);
ltrace!(audit, "Got idl for index {:?} {:?} -> {}", itype, attr, idl);
Ok(Some(idl)) Ok(Some(idl))
})
}) })
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn name2uuid( fn name2uuid(&mut self, name: &str) -> Result<Option<Uuid>, OperationError> {
&mut self,
audit: &mut AuditScope,
name: &str,
) -> Result<Option<Uuid>, OperationError> {
spanned!("be::idl_sqlite::name2uuid", { spanned!("be::idl_sqlite::name2uuid", {
lperf_trace_segment!(audit, "be::idl_sqlite::name2uuid", || { // The table exists - lets now get the actual index itself.
// The table exists - lets now get the actual index itself. let mut stmt = self
let mut stmt = self .get_conn()
.get_conn() .prepare("SELECT uuid FROM idx_name2uuid WHERE name = :name")
.prepare("SELECT uuid FROM idx_name2uuid WHERE name = :name") .map_err(sqlite_error)?;
.map_err(sqlite_error)?; let uuid_raw: Option<String> = stmt
let uuid_raw: Option<String> = stmt .query_row(&[(":name", &name)], |row| row.get(0))
.query_row(&[(":name", &name)], |row| row.get(0)) // We don't mind if it doesn't exist
// We don't mind if it doesn't exist .optional()
.optional() .map_err(sqlite_error)?;
.map_err(sqlite_error)?;
let uuid = uuid_raw.as_ref().and_then(|u| Uuid::parse_str(u).ok()); let uuid = uuid_raw.as_ref().and_then(|u| Uuid::parse_str(u).ok());
trace!(%name, ?uuid, "Got uuid for index"); trace!(%name, ?uuid, "Got uuid for index");
ltrace!(audit, "Got uuid for index name {} -> {:?}", name, uuid);
Ok(uuid) Ok(uuid)
})
}) })
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn uuid2spn( fn uuid2spn(&mut self, uuid: &Uuid) -> Result<Option<Value>, OperationError> {
&mut self,
audit: &mut AuditScope,
uuid: &Uuid,
) -> Result<Option<Value>, OperationError> {
spanned!("be::idl_sqlite::uuid2spn", { spanned!("be::idl_sqlite::uuid2spn", {
lperf_trace_segment!(audit, "be::idl_sqlite::uuid2spn", || { let uuids = uuid.to_hyphenated_ref().to_string();
let uuids = uuid.to_hyphenated_ref().to_string(); // The table exists - lets now get the actual index itself.
// The table exists - lets now get the actual index itself. let mut stmt = self
let mut stmt = self .get_conn()
.get_conn() .prepare("SELECT spn FROM idx_uuid2spn WHERE uuid = :uuid")
.prepare("SELECT spn FROM idx_uuid2spn WHERE uuid = :uuid") .map_err(sqlite_error)?;
.map_err(sqlite_error)?; let spn_raw: Option<Vec<u8>> = stmt
let spn_raw: Option<Vec<u8>> = stmt .query_row(&[(":uuid", &uuids)], |row| row.get(0))
.query_row(&[(":uuid", &uuids)], |row| row.get(0)) // We don't mind if it doesn't exist
// We don't mind if it doesn't exist .optional()
.optional() .map_err(sqlite_error)?;
.map_err(sqlite_error)?;
let spn: Option<Value> = match spn_raw { let spn: Option<Value> = match spn_raw {
Some(d) => { Some(d) => {
let dbv = serde_cbor::from_slice(d.as_slice()).map_err(serde_cbor_error)?; let dbv = serde_cbor::from_slice(d.as_slice()).map_err(serde_cbor_error)?;
let spn = Value::from_db_valuev1(dbv) let spn = Value::from_db_valuev1(dbv)
.map_err(|_| OperationError::CorruptedIndex("uuid2spn".to_string()))?; .map_err(|_| OperationError::CorruptedIndex("uuid2spn".to_string()))?;
Some(spn) Some(spn)
} }
None => None, None => None,
}; };
trace!(?uuid, ?spn, "Got spn for uuid"); trace!(?uuid, ?spn, "Got spn for uuid");
ltrace!(audit, "Got spn for uuid {:?} -> {:?}", uuid, spn);
Ok(spn) Ok(spn)
})
}) })
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn uuid2rdn( fn uuid2rdn(&mut self, uuid: &Uuid) -> Result<Option<String>, OperationError> {
&mut self,
audit: &mut AuditScope,
uuid: &Uuid,
) -> Result<Option<String>, OperationError> {
spanned!("be::idl_sqlite::uuid2rdn", { spanned!("be::idl_sqlite::uuid2rdn", {
lperf_trace_segment!(audit, "be::idl_sqlite::uuid2rdn", || { let uuids = uuid.to_hyphenated_ref().to_string();
let uuids = uuid.to_hyphenated_ref().to_string(); // The table exists - lets now get the actual index itself.
// The table exists - lets now get the actual index itself. let mut stmt = self
let mut stmt = self .get_conn()
.get_conn() .prepare("SELECT rdn FROM idx_uuid2rdn WHERE uuid = :uuid")
.prepare("SELECT rdn FROM idx_uuid2rdn WHERE uuid = :uuid") .map_err(sqlite_error)?;
.map_err(sqlite_error)?; let rdn: Option<String> = stmt
let rdn: Option<String> = stmt .query_row(&[(":uuid", &uuids)], |row| row.get(0))
.query_row(&[(":uuid", &uuids)], |row| row.get(0)) // We don't mind if it doesn't exist
// We don't mind if it doesn't exist .optional()
.optional() .map_err(sqlite_error)?;
.map_err(sqlite_error)?;
trace!(?uuid, ?rdn, "Got rdn for uuid"); trace!(?uuid, ?rdn, "Got rdn for uuid");
ltrace!(audit, "Got rdn for uuid {:?} -> {:?}", uuid, rdn);
Ok(rdn) Ok(rdn)
})
}) })
} }
@ -427,9 +388,8 @@ pub trait IdlSqliteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn get_allids(&self, au: &mut AuditScope) -> Result<IDLBitRange, OperationError> { fn get_allids(&self) -> Result<IDLBitRange, OperationError> {
trace!("Building allids..."); trace!("Building allids...");
ltrace!(au, "Building allids...");
let mut stmt = self let mut stmt = self
.get_conn() .get_conn()
.prepare("SELECT id FROM id2entry") .prepare("SELECT id FROM id2entry")
@ -441,7 +401,6 @@ pub trait IdlSqliteTransaction {
// Convert the idsqlite to id raw // Convert the idsqlite to id raw
id.try_into().map_err(|e| { id.try_into().map_err(|e| {
admin_error!(?e, "I64 Parse Error"); admin_error!(?e, "I64 Parse Error");
ladmin_error!(au, "I64 Parse Error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
}) })
}) })
@ -454,7 +413,7 @@ pub trait IdlSqliteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn list_idxs(&self, _audit: &mut AuditScope) -> Result<Vec<String>, OperationError> { fn list_idxs(&self) -> Result<Vec<String>, OperationError> {
let mut stmt = self let mut stmt = self
.get_conn() .get_conn()
.prepare("SELECT name from sqlite_master where type='table' and name GLOB 'idx_*'") .prepare("SELECT name from sqlite_master where type='table' and name GLOB 'idx_*'")
@ -465,30 +424,23 @@ pub trait IdlSqliteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn list_id2entry(&self, audit: &mut AuditScope) -> Result<Vec<(u64, String)>, OperationError> { fn list_id2entry(&self) -> Result<Vec<(u64, String)>, OperationError> {
let allids = self.get_identry_raw(audit, &IdList::AllIds)?; let allids = self.get_identry_raw(&IdList::AllIds)?;
allids allids
.into_iter() .into_iter()
.map(|data| { .map(|data| data.into_dbentry().map(|(id, db_e)| (id, db_e.to_string())))
data.into_dbentry(audit)
.map(|(id, db_e)| (id, db_e.to_string()))
})
.collect() .collect()
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn get_id2entry( fn get_id2entry(&self, id: u64) -> Result<(u64, String), OperationError> {
&self,
audit: &mut AuditScope,
id: u64,
) -> Result<(u64, String), OperationError> {
let idl = IdList::Indexed(IDLBitRange::from_u64(id)); let idl = IdList::Indexed(IDLBitRange::from_u64(id));
let mut allids = self.get_identry_raw(audit, &idl)?; let mut allids = self.get_identry_raw(&idl)?;
allids allids
.pop() .pop()
.ok_or(OperationError::InvalidEntryId) .ok_or(OperationError::InvalidEntryId)
.and_then(|data| { .and_then(|data| {
data.into_dbentry(audit) data.into_dbentry()
.map(|(id, db_e)| (id, format!("{:?}", db_e))) .map(|(id, db_e)| (id, format!("{:?}", db_e)))
}) })
} }
@ -496,7 +448,6 @@ pub trait IdlSqliteTransaction {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn list_index_content( fn list_index_content(
&self, &self,
_audit: &mut AuditScope,
index_name: &str, index_name: &str,
) -> Result<Vec<(String, IDLBitRange)>, OperationError> { ) -> Result<Vec<(String, IDLBitRange)>, OperationError> {
// TODO: Once we have slopes we can add .exists_table, and assert // TODO: Once we have slopes we can add .exists_table, and assert
@ -624,22 +575,19 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn commit(mut self, audit: &mut AuditScope) -> Result<(), OperationError> { pub fn commit(mut self) -> Result<(), OperationError> {
spanned!("be::idl_sqlite::commit", { spanned!("be::idl_sqlite::commit", {
lperf_trace_segment!(audit, "be::idl_sqlite::commit", || { trace!("Commiting BE WR txn");
// ltrace!(audit, "Commiting BE WR txn"); assert!(!self.committed);
assert!(!self.committed); self.committed = true;
self.committed = true;
self.conn self.conn
.execute("COMMIT TRANSACTION", []) .execute("COMMIT TRANSACTION", [])
.map(|_| ()) .map(|_| ())
.map_err(|e| { .map_err(|e| {
admin_error!(?e, "CRITICAL: failed to commit sqlite txn"); admin_error!(?e, "CRITICAL: failed to commit sqlite txn");
ladmin_error!(audit, "CRITICAL: failed to commit sqlite txn -> {:?}", e); OperationError::BackendEngine
OperationError::BackendEngine })
})
})
}) })
} }
@ -694,7 +642,6 @@ impl IdlSqliteWriteTransaction {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn write_identry( pub fn write_identry(
&self, &self,
au: &mut AuditScope,
entry: &Entry<EntrySealed, EntryCommitted>, entry: &Entry<EntrySealed, EntryCommitted>,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
let dbe = entry.to_dbentry(); let dbe = entry.to_dbentry();
@ -705,15 +652,11 @@ impl IdlSqliteWriteTransaction {
data, data,
}); });
self.write_identries_raw(au, raw_entries) self.write_identries_raw(raw_entries)
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn write_identries_raw<I>( pub fn write_identries_raw<I>(&self, mut entries: I) -> Result<(), OperationError>
&self,
_au: &mut AuditScope,
mut entries: I,
) -> Result<(), OperationError>
where where
I: Iterator<Item = IdRawEntry>, I: Iterator<Item = IdRawEntry>,
{ {
@ -773,8 +716,7 @@ impl IdlSqliteWriteTransaction {
*/ */
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn delete_identry(&self, _au: &mut AuditScope, id: u64) -> Result<(), OperationError> { pub fn delete_identry(&self, id: u64) -> Result<(), OperationError> {
// lperf_trace_segment!(au, "be::idl_sqlite::delete_identry", || {
let mut stmt = self let mut stmt = self
.conn .conn
.prepare("DELETE FROM id2entry WHERE id = :id") .prepare("DELETE FROM id2entry WHERE id = :id")
@ -794,66 +736,60 @@ impl IdlSqliteWriteTransaction {
debug_assert!(iid > 0); debug_assert!(iid > 0);
stmt.execute(&[&iid]).map(|_| ()).map_err(sqlite_error) stmt.execute(&[&iid]).map(|_| ()).map_err(sqlite_error)
// })
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn write_idl( pub fn write_idl(
&self, &self,
audit: &mut AuditScope,
attr: &str, attr: &str,
itype: &IndexType, itype: &IndexType,
idx_key: &str, idx_key: &str,
idl: &IDLBitRange, idl: &IDLBitRange,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
spanned!("be::idl_sqlite::write_idl", { spanned!("be::idl_sqlite::write_idl", {
lperf_trace_segment!(audit, "be::idl_sqlite::write_idl", || { if idl.is_empty() {
if idl.is_empty() { trace!(?idl, "purging idl");
trace!(?idl, "purging idl"); // delete it
ltrace!(audit, "purging idl -> {:?}", idl); // Delete this idx_key from the table.
// delete it let query = format!(
// Delete this idx_key from the table. "DELETE FROM idx_{}_{} WHERE key = :key",
let query = format!( itype.as_idx_str(),
"DELETE FROM idx_{}_{} WHERE key = :key", attr
itype.as_idx_str(), );
attr
);
self.conn self.conn
.prepare(query.as_str()) .prepare(query.as_str())
.and_then(|mut stmt| stmt.execute(&[(":key", &idx_key)])) .and_then(|mut stmt| stmt.execute(&[(":key", &idx_key)]))
.map_err(sqlite_error) .map_err(sqlite_error)
} else { } else {
trace!(?idl, "writing idl"); trace!(?idl, "writing idl");
ltrace!(audit, "writing idl -> {}", idl); // Serialise the IdList to Vec<u8>
// Serialise the IdList to Vec<u8> let idl_raw = serde_cbor::to_vec(idl).map_err(serde_cbor_error)?;
let idl_raw = serde_cbor::to_vec(idl).map_err(serde_cbor_error)?;
// update or create it. // update or create it.
let query = format!( let query = format!(
"INSERT OR REPLACE INTO idx_{}_{} (key, idl) VALUES(:key, :idl)", "INSERT OR REPLACE INTO idx_{}_{} (key, idl) VALUES(:key, :idl)",
itype.as_idx_str(), itype.as_idx_str(),
attr attr
); );
self.conn self.conn
.prepare(query.as_str()) .prepare(query.as_str())
.and_then(|mut stmt| { .and_then(|mut stmt| {
stmt.execute(named_params! { stmt.execute(named_params! {
":key": &idx_key, ":key": &idx_key,
":idl": &idl_raw ":idl": &idl_raw
})
}) })
.map_err(sqlite_error) })
} .map_err(sqlite_error)
// Get rid of the sqlite rows usize }
.map(|_| ()) // Get rid of the sqlite rows usize
}) .map(|_| ())
}) })
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn create_name2uuid(&self, _audit: &mut AuditScope) -> Result<(), OperationError> { pub fn create_name2uuid(&self) -> Result<(), OperationError> {
self.conn self.conn
.execute( .execute(
"CREATE TABLE IF NOT EXISTS idx_name2uuid (name TEXT PRIMARY KEY, uuid TEXT)", "CREATE TABLE IF NOT EXISTS idx_name2uuid (name TEXT PRIMARY KEY, uuid TEXT)",
@ -864,12 +800,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn write_name2uuid_add( pub fn write_name2uuid_add(&self, name: &str, uuid: &Uuid) -> Result<(), OperationError> {
&self,
_audit: &mut AuditScope,
name: &str,
uuid: &Uuid,
) -> Result<(), OperationError> {
let uuids = uuid.to_hyphenated_ref().to_string(); let uuids = uuid.to_hyphenated_ref().to_string();
self.conn self.conn
@ -885,11 +816,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn write_name2uuid_rem( pub fn write_name2uuid_rem(&self, name: &str) -> Result<(), OperationError> {
&self,
_audit: &mut AuditScope,
name: &str,
) -> Result<(), OperationError> {
self.conn self.conn
.prepare("DELETE FROM idx_name2uuid WHERE name = :name") .prepare("DELETE FROM idx_name2uuid WHERE name = :name")
.and_then(|mut stmt| stmt.execute(&[(":name", &name)])) .and_then(|mut stmt| stmt.execute(&[(":name", &name)]))
@ -898,7 +825,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn create_uuid2spn(&self, _audit: &mut AuditScope) -> Result<(), OperationError> { pub fn create_uuid2spn(&self) -> Result<(), OperationError> {
self.conn self.conn
.execute( .execute(
"CREATE TABLE IF NOT EXISTS idx_uuid2spn (uuid TEXT PRIMARY KEY, spn BLOB)", "CREATE TABLE IF NOT EXISTS idx_uuid2spn (uuid TEXT PRIMARY KEY, spn BLOB)",
@ -909,12 +836,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn write_uuid2spn( pub fn write_uuid2spn(&self, uuid: &Uuid, k: Option<&Value>) -> Result<(), OperationError> {
&self,
_audit: &mut AuditScope,
uuid: &Uuid,
k: Option<&Value>,
) -> Result<(), OperationError> {
let uuids = uuid.to_hyphenated_ref().to_string(); let uuids = uuid.to_hyphenated_ref().to_string();
match k { match k {
Some(k) => { Some(k) => {
@ -941,7 +863,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn create_uuid2rdn(&self, _audit: &mut AuditScope) -> Result<(), OperationError> { pub fn create_uuid2rdn(&self) -> Result<(), OperationError> {
self.conn self.conn
.execute( .execute(
"CREATE TABLE IF NOT EXISTS idx_uuid2rdn (uuid TEXT PRIMARY KEY, rdn TEXT)", "CREATE TABLE IF NOT EXISTS idx_uuid2rdn (uuid TEXT PRIMARY KEY, rdn TEXT)",
@ -952,12 +874,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn write_uuid2rdn( pub fn write_uuid2rdn(&self, uuid: &Uuid, k: Option<&String>) -> Result<(), OperationError> {
&self,
_audit: &mut AuditScope,
uuid: &Uuid,
k: Option<&String>,
) -> Result<(), OperationError> {
let uuids = uuid.to_hyphenated_ref().to_string(); let uuids = uuid.to_hyphenated_ref().to_string();
match k { match k {
Some(k) => self Some(k) => self
@ -976,12 +893,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn create_idx( pub fn create_idx(&self, attr: &str, itype: &IndexType) -> Result<(), OperationError> {
&self,
audit: &mut AuditScope,
attr: &str,
itype: &IndexType,
) -> Result<(), OperationError> {
// Is there a better way than formatting this? I can't seem // Is there a better way than formatting this? I can't seem
// to template into the str. // to template into the str.
// //
@ -992,7 +904,6 @@ impl IdlSqliteWriteTransaction {
attr attr
); );
trace!(idx = %idx_stmt, "Creating index"); trace!(idx = %idx_stmt, "Creating index");
ltrace!(audit, "Creating index -> {}", idx_stmt);
self.conn self.conn
.execute(idx_stmt.as_str(), []) .execute(idx_stmt.as_str(), [])
@ -1001,12 +912,11 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub unsafe fn purge_idxs(&self, audit: &mut AuditScope) -> Result<(), OperationError> { pub unsafe fn purge_idxs(&self) -> Result<(), OperationError> {
let idx_table_list = self.list_idxs(audit)?; let idx_table_list = self.list_idxs()?;
idx_table_list.iter().try_for_each(|idx_table| { idx_table_list.iter().try_for_each(|idx_table| {
trace!(table = ?idx_table, "removing idx_table"); trace!(table = ?idx_table, "removing idx_table");
ltrace!(audit, "removing idx_table -> {:?}", idx_table);
self.conn self.conn
.prepare(format!("DROP TABLE {}", idx_table).as_str()) .prepare(format!("DROP TABLE {}", idx_table).as_str())
.and_then(|mut stmt| stmt.execute([]).map(|_| ())) .and_then(|mut stmt| stmt.execute([]).map(|_| ()))
@ -1017,7 +927,6 @@ impl IdlSqliteWriteTransaction {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn store_idx_slope_analysis( pub fn store_idx_slope_analysis(
&self, &self,
_audit: &mut AuditScope,
slopes: &HashMap<IdxKey, IdxSlope>, slopes: &HashMap<IdxKey, IdxSlope>,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
self.conn self.conn
@ -1057,20 +966,13 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn is_idx_slopeyness_generated( pub fn is_idx_slopeyness_generated(&self) -> Result<bool, OperationError> {
&self, self.exists_table("idxslope_analysis")
audit: &mut AuditScope,
) -> Result<bool, OperationError> {
self.exists_table(audit, "idxslope_analysis")
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn get_idx_slope( pub fn get_idx_slope(&self, ikey: &IdxKey) -> Result<Option<IdxSlope>, OperationError> {
&self, let analysis_exists = self.exists_table("idxslope_analysis")?;
audit: &mut AuditScope,
ikey: &IdxKey,
) -> Result<Option<IdxSlope>, OperationError> {
let analysis_exists = self.exists_table(audit, "idxslope_analysis")?;
if !analysis_exists { if !analysis_exists {
return Ok(None); return Ok(None);
} }
@ -1090,15 +992,13 @@ impl IdlSqliteWriteTransaction {
.optional() .optional()
.map_err(sqlite_error)?; .map_err(sqlite_error)?;
trace!(name = %key, ?slope, "Got slope for index"); trace!(name = %key, ?slope, "Got slope for index");
ltrace!(audit, "Got slope for index name {} -> {:?}", key, slope);
Ok(slope) Ok(slope)
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub unsafe fn purge_id2entry(&self, audit: &mut AuditScope) -> Result<(), OperationError> { pub unsafe fn purge_id2entry(&self) -> Result<(), OperationError> {
trace!("purge id2entry ..."); trace!("purge id2entry ...");
ltrace!(audit, "purge id2entry ...");
self.conn self.conn
.execute("DELETE FROM id2entry", []) .execute("DELETE FROM id2entry", [])
.map(|_| ()) .map(|_| ())
@ -1247,7 +1147,7 @@ impl IdlSqliteWriteTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn setup(&self, audit: &mut AuditScope) -> Result<(), OperationError> { pub fn setup(&self) -> Result<(), OperationError> {
// This stores versions of components. For example: // This stores versions of components. For example:
// ---------------------- // ----------------------
// | id | version | // | id | version |
@ -1274,7 +1174,6 @@ impl IdlSqliteWriteTransaction {
// If the table is empty, populate the versions as 0. // If the table is empty, populate the versions as 0.
let mut dbv_id2entry = self.get_db_version_key(DBV_ID2ENTRY); let mut dbv_id2entry = self.get_db_version_key(DBV_ID2ENTRY);
trace!(initial = %dbv_id2entry, "dbv_id2entry"); trace!(initial = %dbv_id2entry, "dbv_id2entry");
ltrace!(audit, "dbv_id2entry initial == {}", dbv_id2entry);
// Check db_version here. // Check db_version here.
// * if 0 -> create v1. // * if 0 -> create v1.
@ -1303,11 +1202,6 @@ impl IdlSqliteWriteTransaction {
dbv_id2entry = 1; dbv_id2entry = 1;
admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (id2entry, db_sid)"); admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (id2entry, db_sid)");
ladmin_info!(
audit,
"dbv_id2entry migrated (id2entry, db_sid) -> {}",
dbv_id2entry
);
} }
// * if v1 -> add the domain uuid table // * if v1 -> add the domain uuid table
if dbv_id2entry == 1 { if dbv_id2entry == 1 {
@ -1324,7 +1218,6 @@ impl IdlSqliteWriteTransaction {
dbv_id2entry = 2; dbv_id2entry = 2;
admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (db_did)"); admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (db_did)");
ladmin_info!(audit, "dbv_id2entry migrated (db_did) -> {}", dbv_id2entry);
} }
// * if v2 -> add the op max ts table. // * if v2 -> add the op max ts table.
if dbv_id2entry == 2 { if dbv_id2entry == 2 {
@ -1340,24 +1233,14 @@ impl IdlSqliteWriteTransaction {
.map_err(sqlite_error)?; .map_err(sqlite_error)?;
dbv_id2entry = 3; dbv_id2entry = 3;
admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (db_op_ts)"); admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (db_op_ts)");
ladmin_info!(
audit,
"dbv_id2entry migrated (db_op_ts) -> {}",
dbv_id2entry
);
} }
// * if v3 -> create name2uuid, uuid2spn, uuid2rdn. // * if v3 -> create name2uuid, uuid2spn, uuid2rdn.
if dbv_id2entry == 3 { if dbv_id2entry == 3 {
self.create_name2uuid(audit) self.create_name2uuid()
.and_then(|_| self.create_uuid2spn(audit)) .and_then(|_| self.create_uuid2spn())
.and_then(|_| self.create_uuid2rdn(audit))?; .and_then(|_| self.create_uuid2rdn())?;
dbv_id2entry = 4; dbv_id2entry = 4;
admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (name2uuid, uuid2spn, uuid2rdn)"); admin_info!(entry = %dbv_id2entry, "dbv_id2entry migrated (name2uuid, uuid2spn, uuid2rdn)");
ladmin_info!(
audit,
"dbv_id2entry migrated (name2uuid, uuid2spn, uuid2rdn) -> {}",
dbv_id2entry
);
} }
// * if v4 -> complete. // * if v4 -> complete.
@ -1374,11 +1257,7 @@ impl IdlSqliteWriteTransaction {
impl IdlSqlite { impl IdlSqlite {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn new( pub fn new(cfg: &BackendConfig, vacuum: bool) -> Result<Self, OperationError> {
audit: &mut AuditScope,
cfg: &BackendConfig,
vacuum: bool,
) -> Result<Self, OperationError> {
if cfg.path.is_empty() { if cfg.path.is_empty() {
debug_assert!(cfg.pool_size == 1); debug_assert!(cfg.pool_size == 1);
} }
@ -1396,10 +1275,12 @@ impl IdlSqlite {
immediate = true, immediate = true,
"NOTICE: A db vacuum has been requested. This may take a long time ..." "NOTICE: A db vacuum has been requested. This may take a long time ..."
); );
/*
limmediate_warning!( limmediate_warning!(
audit, audit,
"NOTICE: A db vacuum has been requested. This may take a long time ...\n" "NOTICE: A db vacuum has been requested. This may take a long time ...\n"
); );
*/
let vconn = let vconn =
Connection::open_with_flags(cfg.path.as_str(), flags).map_err(sqlite_error)?; Connection::open_with_flags(cfg.path.as_str(), flags).map_err(sqlite_error)?;
@ -1408,7 +1289,6 @@ impl IdlSqlite {
.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") .execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")
.map_err(|e| { .map_err(|e| {
admin_error!(?e, "rusqlite wal_checkpoint error"); admin_error!(?e, "rusqlite wal_checkpoint error");
ladmin_error!(audit, "rusqlite wal_checkpoint error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
@ -1416,13 +1296,11 @@ impl IdlSqlite {
.pragma_update(None, "journal_mode", &"DELETE") .pragma_update(None, "journal_mode", &"DELETE")
.map_err(|e| { .map_err(|e| {
admin_error!(?e, "rusqlite journal_mode update error"); admin_error!(?e, "rusqlite journal_mode update error");
ladmin_error!(audit, "rusqlite journal_mode update error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
vconn.close().map_err(|e| { vconn.close().map_err(|e| {
admin_error!(?e, "rusqlite db close error"); admin_error!(?e, "rusqlite db close error");
ladmin_error!(audit, "rusqlite db close error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
@ -1433,13 +1311,11 @@ impl IdlSqlite {
.pragma_update(None, "page_size", &(cfg.fstype as u32)) .pragma_update(None, "page_size", &(cfg.fstype as u32))
.map_err(|e| { .map_err(|e| {
admin_error!(?e, "rusqlite page_size update error"); admin_error!(?e, "rusqlite page_size update error");
ladmin_error!(audit, "rusqlite page_size update error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
vconn.execute_batch("VACUUM").map_err(|e| { vconn.execute_batch("VACUUM").map_err(|e| {
admin_error!(?e, "rusqlite vacuum error"); admin_error!(?e, "rusqlite vacuum error");
ladmin_error!(audit, "rusqlite vacuum error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
@ -1447,18 +1323,16 @@ impl IdlSqlite {
.pragma_update(None, "journal_mode", &"WAL") .pragma_update(None, "journal_mode", &"WAL")
.map_err(|e| { .map_err(|e| {
admin_error!(?e, "rusqlite journal_mode update error"); admin_error!(?e, "rusqlite journal_mode update error");
ladmin_error!(audit, "rusqlite journal_mode update error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
vconn.close().map_err(|e| { vconn.close().map_err(|e| {
admin_error!(?e, "rusqlite db close error"); admin_error!(?e, "rusqlite db close error");
ladmin_error!(audit, "rusqlite db close error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
admin_warn!(immediate = true, "NOTICE: db vacuum complete"); admin_warn!(immediate = true, "NOTICE: db vacuum complete");
limmediate_warning!(audit, "NOTICE: db vacuum complete\n"); // limmediate_warning!(audit, "NOTICE: db vacuum complete\n");
}; };
let fs_page_size = cfg.fstype as u32; let fs_page_size = cfg.fstype as u32;
@ -1484,7 +1358,7 @@ impl IdlSqlite {
// Look at max_size and thread_pool here for perf later // Look at max_size and thread_pool here for perf later
let pool = builder2.build(manager).map_err(|e| { let pool = builder2.build(manager).map_err(|e| {
admin_error!(?e, "r2d2 error"); admin_error!(?e, "r2d2 error");
ladmin_error!(audit, "r2d2 error {:?}", e); // ladmin_error!(audit, "r2d2 error {:?}", e);
OperationError::SqliteError OperationError::SqliteError
})?; })?;
@ -1492,9 +1366,8 @@ impl IdlSqlite {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub(crate) fn get_allids_count(&self, au: &mut AuditScope) -> Result<u64, OperationError> { pub(crate) fn get_allids_count(&self) -> Result<u64, OperationError> {
trace!("Counting allids..."); trace!("Counting allids...");
ltrace!(au, "Counting allids...");
#[allow(clippy::expect_used)] #[allow(clippy::expect_used)]
self.pool self.pool
.try_get() .try_get()
@ -1527,15 +1400,14 @@ impl IdlSqlite {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::audit::AuditScope;
use crate::be::idl_sqlite::{IdlSqlite, IdlSqliteTransaction}; use crate::be::idl_sqlite::{IdlSqlite, IdlSqliteTransaction};
use crate::be::BackendConfig; use crate::be::BackendConfig;
#[test] #[test]
fn test_idl_sqlite_verify() { fn test_idl_sqlite_verify() {
let mut audit = AuditScope::new("run_test", uuid::Uuid::new_v4(), None); let _ = crate::tracing_tree::test_init();
let cfg = BackendConfig::new_test(); let cfg = BackendConfig::new_test();
let be = IdlSqlite::new(&mut audit, &cfg, false).unwrap(); let be = IdlSqlite::new(&cfg, false).unwrap();
let be_w = be.write(); let be_w = be.write();
let r = be_w.verify(); let r = be_w.verify();
assert!(r.len() == 0); assert!(r.len() == 0);

402
kanidmd/src/lib/be/mod.rs
View file

@ -125,29 +125,23 @@ pub struct BackendWriteTransaction<'a> {
impl IdRawEntry { impl IdRawEntry {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn into_dbentry(self, audit: &mut AuditScope) -> Result<(u64, DbEntry), OperationError> { fn into_dbentry(self) -> Result<(u64, DbEntry), OperationError> {
serde_cbor::from_slice(self.data.as_slice()) serde_cbor::from_slice(self.data.as_slice())
.map_err(|e| { .map_err(|e| {
admin_error!(?e, "Serde CBOR Error"); admin_error!(?e, "Serde CBOR Error");
ladmin_error!(audit, "Serde CBOR Error -> {:?}", e);
OperationError::SerdeCborError OperationError::SerdeCborError
}) })
.map(|dbe| (self.id, dbe)) .map(|dbe| (self.id, dbe))
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn into_entry( fn into_entry(self) -> Result<Entry<EntrySealed, EntryCommitted>, OperationError> {
self,
audit: &mut AuditScope,
) -> Result<Entry<EntrySealed, EntryCommitted>, OperationError> {
let db_e = serde_cbor::from_slice(self.data.as_slice()).map_err(|e| { let db_e = serde_cbor::from_slice(self.data.as_slice()).map_err(|e| {
admin_error!(?e, "Serde CBOR Error"); admin_error!(?e, "Serde CBOR Error");
ladmin_error!(audit, "Serde CBOR Error -> {:?}", e);
OperationError::SerdeCborError OperationError::SerdeCborError
})?; })?;
// let id = u64::try_from(self.id).map_err(|_| OperationError::InvalidEntryId)?; // let id = u64::try_from(self.id).map_err(|_| OperationError::InvalidEntryId)?;
Entry::from_dbentry(audit, db_e, self.id) Entry::from_dbentry(db_e, self.id).ok_or_else(|| OperationError::CorruptedEntry(self.id))
.map_err(|_| OperationError::CorruptedEntry(self.id))
} }
} }
@ -177,7 +171,7 @@ pub trait BackendTransaction {
// Get the idl for this // Get the idl for this
match self match self
.get_idlayer() .get_idlayer()
.get_idl(au, attr, &IndexType::Equality, &idx_key)? .get_idl(attr, &IndexType::Equality, &idx_key)?
{ {
Some(idl) => ( Some(idl) => (
IdList::Indexed(idl), IdList::Indexed(idl),
@ -197,7 +191,7 @@ pub trait BackendTransaction {
// Get the idl for this // Get the idl for this
match self match self
.get_idlayer() .get_idlayer()
.get_idl(au, attr, &IndexType::SubString, &idx_key)? .get_idl(attr, &IndexType::SubString, &idx_key)?
{ {
Some(idl) => ( Some(idl) => (
IdList::Indexed(idl), IdList::Indexed(idl),
@ -214,7 +208,6 @@ pub trait BackendTransaction {
if idx.is_some() { if idx.is_some() {
// Get the idl for this // Get the idl for this
match self.get_idlayer().get_idl( match self.get_idlayer().get_idl(
au,
attr, attr,
&IndexType::Presence, &IndexType::Presence,
&"_".to_string(), &"_".to_string(),
@ -609,7 +602,7 @@ pub trait BackendTransaction {
} }
}; };
let entries = self.get_idlayer().get_identry(au, &idl).map_err(|e| { let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
admin_error!(?e, "get_identry failed"); admin_error!(?e, "get_identry failed");
ladmin_error!(au, "get_identry failed {:?}", e); ladmin_error!(au, "get_identry failed {:?}", e);
e e
@ -725,7 +718,7 @@ pub trait BackendTransaction {
match &idl { match &idl {
IdList::Indexed(idl) => Ok(!idl.is_empty()), IdList::Indexed(idl) => Ok(!idl.is_empty()),
_ => { _ => {
let entries = self.get_idlayer().get_identry(au, &idl).map_err(|e| { let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
admin_error!(?e, "get_identry failed"); admin_error!(?e, "get_identry failed");
ladmin_error!(au, "get_identry failed {:?}", e); ladmin_error!(au, "get_identry failed {:?}", e);
e e
@ -754,8 +747,8 @@ pub trait BackendTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn verify(&self, audit: &mut AuditScope) -> Vec<Result<(), ConsistencyError>> { fn verify(&self) -> Vec<Result<(), ConsistencyError>> {
self.get_idlayer().verify(audit) self.get_idlayer().verify()
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
@ -780,8 +773,9 @@ pub trait BackendTransaction {
}; };
// If the set.len > 1, check each item. // If the set.len > 1, check each item.
n2u_set.iter().try_for_each(|name| { n2u_set
match self.get_idlayer().name2uuid(audit, name) { .iter()
.try_for_each(|name| match self.get_idlayer().name2uuid(name) {
Ok(Some(idx_uuid)) => { Ok(Some(idx_uuid)) => {
if &idx_uuid == e_uuid { if &idx_uuid == e_uuid {
Ok(()) Ok(())
@ -799,11 +793,10 @@ pub trait BackendTransaction {
ladmin_error!(audit, "Invalid name2uuid state -> {:?}", r); ladmin_error!(audit, "Invalid name2uuid state -> {:?}", r);
Err(ConsistencyError::BackendIndexSync) Err(ConsistencyError::BackendIndexSync)
} }
} })?;
})?;
let spn = e.get_uuid2spn(); let spn = e.get_uuid2spn();
match self.get_idlayer().uuid2spn(audit, &e_uuid) { match self.get_idlayer().uuid2spn(&e_uuid) {
Ok(Some(idx_spn)) => { Ok(Some(idx_spn)) => {
if spn != idx_spn { if spn != idx_spn {
admin_error!("Invalid uuid2spn state -> incorrect idx spn value"); admin_error!("Invalid uuid2spn state -> incorrect idx spn value");
@ -819,7 +812,7 @@ pub trait BackendTransaction {
}; };
let rdn = e.get_uuid2rdn(); let rdn = e.get_uuid2rdn();
match self.get_idlayer().uuid2rdn(audit, &e_uuid) { match self.get_idlayer().uuid2rdn(&e_uuid) {
Ok(Some(idx_rdn)) => { Ok(Some(idx_rdn)) => {
if rdn != idx_rdn { if rdn != idx_rdn {
admin_error!("Invalid uuid2rdn state -> incorrect idx rdn value"); admin_error!("Invalid uuid2rdn state -> incorrect idx rdn value");
@ -847,7 +840,7 @@ pub trait BackendTransaction {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn verify_indexes(&self, audit: &mut AuditScope) -> Vec<Result<(), ConsistencyError>> { fn verify_indexes(&self, audit: &mut AuditScope) -> Vec<Result<(), ConsistencyError>> {
let idl = IdList::AllIds; let idl = IdList::AllIds;
let entries = match self.get_idlayer().get_identry(audit, &idl) { let entries = match self.get_idlayer().get_identry(&idl) {
Ok(s) => s, Ok(s) => s,
Err(e) => { Err(e) => {
admin_error!(?e, "get_identry failure"); admin_error!(?e, "get_identry failure");
@ -872,7 +865,7 @@ pub trait BackendTransaction {
// load all entries into RAM, may need to change this later // load all entries into RAM, may need to change this later
// if the size of the database compared to RAM is an issue // if the size of the database compared to RAM is an issue
let idl = IdList::AllIds; let idl = IdList::AllIds;
let raw_entries: Vec<IdRawEntry> = self.get_idlayer().get_identry_raw(audit, &idl)?; let raw_entries: Vec<IdRawEntry> = self.get_idlayer().get_identry_raw(&idl)?;
let entries: Result<Vec<DbEntry>, _> = raw_entries let entries: Result<Vec<DbEntry>, _> = raw_entries
.iter() .iter()
@ -900,30 +893,18 @@ pub trait BackendTransaction {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn name2uuid( fn name2uuid(&self, name: &str) -> Result<Option<Uuid>, OperationError> {
&self, self.get_idlayer().name2uuid(name)
audit: &mut AuditScope,
name: &str,
) -> Result<Option<Uuid>, OperationError> {
self.get_idlayer().name2uuid(audit, name)
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn uuid2spn( fn uuid2spn(&self, uuid: &Uuid) -> Result<Option<Value>, OperationError> {
&self, self.get_idlayer().uuid2spn(uuid)
audit: &mut AuditScope,
uuid: &Uuid,
) -> Result<Option<Value>, OperationError> {
self.get_idlayer().uuid2spn(audit, uuid)
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn uuid2rdn( fn uuid2rdn(&self, uuid: &Uuid) -> Result<Option<String>, OperationError> {
&self, self.get_idlayer().uuid2rdn(uuid)
audit: &mut AuditScope,
uuid: &Uuid,
) -> Result<Option<String>, OperationError> {
self.get_idlayer().uuid2rdn(audit, uuid)
} }
} }
@ -953,34 +934,26 @@ impl<'a> BackendTransaction for BackendReadTransaction<'a> {
impl<'a> BackendReadTransaction<'a> { impl<'a> BackendReadTransaction<'a> {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn list_indexes(&self, audit: &mut AuditScope) -> Result<Vec<String>, OperationError> { pub fn list_indexes(&self) -> Result<Vec<String>, OperationError> {
self.get_idlayer().list_idxs(audit) self.get_idlayer().list_idxs()
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn list_id2entry( pub fn list_id2entry(&self) -> Result<Vec<(u64, String)>, OperationError> {
&self, self.get_idlayer().list_id2entry()
audit: &mut AuditScope,
) -> Result<Vec<(u64, String)>, OperationError> {
self.get_idlayer().list_id2entry(audit)
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn list_index_content( pub fn list_index_content(
&self, &self,
audit: &mut AuditScope,
index_name: &str, index_name: &str,
) -> Result<Vec<(String, IDLBitRange)>, OperationError> { ) -> Result<Vec<(String, IDLBitRange)>, OperationError> {
self.get_idlayer().list_index_content(audit, index_name) self.get_idlayer().list_index_content(index_name)
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
pub fn get_id2entry( pub fn get_id2entry(&self, id: u64) -> Result<(u64, String), OperationError> {
&self, self.get_idlayer().get_id2entry(id)
audit: &mut AuditScope,
id: u64,
) -> Result<(u64, String), OperationError> {
self.get_idlayer().get_id2entry(audit, id)
} }
} }
@ -1025,7 +998,7 @@ impl<'a> BackendWriteTransaction<'a> {
}) })
.collect(); .collect();
idlayer.write_identries(au, c_entries.iter())?; idlayer.write_identries(c_entries.iter())?;
idlayer.set_id2entry_max_id(id_max); idlayer.set_id2entry_max_id(id_max);
@ -1087,8 +1060,7 @@ impl<'a> BackendWriteTransaction<'a> {
*/ */
// Now, given the list of id's, update them // Now, given the list of id's, update them
self.get_idlayer() self.get_idlayer().write_identries(post_entries.iter())?;
.write_identries(au, post_entries.iter())?;
// Finally, we now reindex all the changed entries. We do this by iterating and zipping // Finally, we now reindex all the changed entries. We do this by iterating and zipping
// over the set, because we know the list is in the same order. // over the set, because we know the list is in the same order.
@ -1117,7 +1089,7 @@ impl<'a> BackendWriteTransaction<'a> {
let id_list = entries.iter().map(|e| e.get_id()); let id_list = entries.iter().map(|e| e.get_id());
// Now, given the list of id's, delete them. // Now, given the list of id's, delete them.
self.get_idlayer().delete_identry(au, id_list)?; self.get_idlayer().delete_identry(id_list)?;
// Finally, purge the indexes from the entries we removed. // Finally, purge the indexes from the entries we removed.
entries entries
@ -1131,7 +1103,7 @@ impl<'a> BackendWriteTransaction<'a> {
audit: &mut AuditScope, audit: &mut AuditScope,
idxkeys: Vec<IdxKey>, idxkeys: Vec<IdxKey>,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
if self.is_idx_slopeyness_generated(audit)? { if self.is_idx_slopeyness_generated()? {
ltrace!(audit, "Indexing slopes available"); ltrace!(audit, "Indexing slopes available");
} else { } else {
ladmin_warning!( ladmin_warning!(
@ -1227,19 +1199,19 @@ impl<'a> BackendWriteTransaction<'a> {
// Write the changes out to the backend // Write the changes out to the backend
if let Some(rem) = n2u_rem { if let Some(rem) = n2u_rem {
idlayer.write_name2uuid_rem(audit, rem)? idlayer.write_name2uuid_rem(rem)?
} }
match u2s_act { match u2s_act {
None => {} None => {}
Some(Ok(k)) => idlayer.write_uuid2spn(audit, uuid, Some(k))?, Some(Ok(k)) => idlayer.write_uuid2spn(uuid, Some(k))?,
Some(Err(_)) => idlayer.write_uuid2spn(audit, uuid, None)?, Some(Err(_)) => idlayer.write_uuid2spn(uuid, None)?,
} }
match u2r_act { match u2r_act {
None => {} None => {}
Some(Ok(k)) => idlayer.write_uuid2rdn(audit, uuid, Some(k))?, Some(Ok(k)) => idlayer.write_uuid2rdn(uuid, Some(k))?,
Some(Err(_)) => idlayer.write_uuid2rdn(audit, uuid, None)?, Some(Err(_)) => idlayer.write_uuid2rdn(uuid, None)?,
} }
// Return none, mask_pre is now completed. // Return none, mask_pre is now completed.
None None
@ -1261,22 +1233,22 @@ impl<'a> BackendWriteTransaction<'a> {
// Write the changes out to the backend // Write the changes out to the backend
if let Some(add) = n2u_add { if let Some(add) = n2u_add {
idlayer.write_name2uuid_add(audit, e_uuid, add)? idlayer.write_name2uuid_add(e_uuid, add)?
} }
if let Some(rem) = n2u_rem { if let Some(rem) = n2u_rem {
idlayer.write_name2uuid_rem(audit, rem)? idlayer.write_name2uuid_rem(rem)?
} }
match u2s_act { match u2s_act {
None => {} None => {}
Some(Ok(k)) => idlayer.write_uuid2spn(audit, e_uuid, Some(k))?, Some(Ok(k)) => idlayer.write_uuid2spn(e_uuid, Some(k))?,
Some(Err(_)) => idlayer.write_uuid2spn(audit, e_uuid, None)?, Some(Err(_)) => idlayer.write_uuid2spn(e_uuid, None)?,
} }
match u2r_act { match u2r_act {
None => {} None => {}
Some(Ok(k)) => idlayer.write_uuid2rdn(audit, e_uuid, Some(k))?, Some(Ok(k)) => idlayer.write_uuid2rdn(e_uuid, Some(k))?,
Some(Err(_)) => idlayer.write_uuid2rdn(audit, e_uuid, None)?, Some(Err(_)) => idlayer.write_uuid2rdn(e_uuid, None)?,
} }
// Extremely Cursed - Okay, we know that self.idxmeta will NOT be changed // Extremely Cursed - Okay, we know that self.idxmeta will NOT be changed
@ -1294,10 +1266,10 @@ impl<'a> BackendWriteTransaction<'a> {
match act { match act {
Ok((attr, itype, idx_key)) => { Ok((attr, itype, idx_key)) => {
ltrace!(audit, "Adding {:?} idx -> {:?}: {:?}", itype, attr, idx_key); ltrace!(audit, "Adding {:?} idx -> {:?}: {:?}", itype, attr, idx_key);
match idlayer.get_idl(audit, attr, itype, idx_key)? { match idlayer.get_idl(attr, itype, idx_key)? {
Some(mut idl) => { Some(mut idl) => {
idl.insert_id(e_id); idl.insert_id(e_id);
idlayer.write_idl(audit, attr, itype, idx_key, &idl) idlayer.write_idl(attr, itype, idx_key, &idl)
} }
None => { None => {
ladmin_error!( ladmin_error!(
@ -1311,10 +1283,10 @@ impl<'a> BackendWriteTransaction<'a> {
} }
Err((attr, itype, idx_key)) => { Err((attr, itype, idx_key)) => {
ltrace!(audit, "Removing {:?} idx -> {:?}: {:?}", itype, attr, idx_key); ltrace!(audit, "Removing {:?} idx -> {:?}: {:?}", itype, attr, idx_key);
match idlayer.get_idl(audit, attr, itype, idx_key)? { match idlayer.get_idl(attr, itype, idx_key)? {
Some(mut idl) => { Some(mut idl) => {
idl.remove_id(e_id); idl.remove_id(e_id);
idlayer.write_idl(audit, attr, itype, idx_key, &idl) idlayer.write_idl(attr, itype, idx_key, &idl)
} }
None => { None => {
ladmin_error!( ladmin_error!(
@ -1336,7 +1308,7 @@ impl<'a> BackendWriteTransaction<'a> {
&self, &self,
audit: &mut AuditScope, audit: &mut AuditScope,
) -> Result<Vec<(AttrString, IndexType)>, OperationError> { ) -> Result<Vec<(AttrString, IndexType)>, OperationError> {
let idx_table_list = self.get_idlayer().list_idxs(audit)?; let idx_table_list = self.get_idlayer().list_idxs()?;
// Turn the vec to a real set // Turn the vec to a real set
let idx_table_set: HashSet<_> = idx_table_list.into_iter().collect(); let idx_table_set: HashSet<_> = idx_table_list.into_iter().collect();
@ -1364,18 +1336,18 @@ impl<'a> BackendWriteTransaction<'a> {
let idlayer = self.get_idlayer(); let idlayer = self.get_idlayer();
// Create name2uuid and uuid2name // Create name2uuid and uuid2name
ltrace!(audit, "Creating index -> name2uuid"); ltrace!(audit, "Creating index -> name2uuid");
idlayer.create_name2uuid(audit)?; idlayer.create_name2uuid()?;
ltrace!(audit, "Creating index -> uuid2spn"); ltrace!(audit, "Creating index -> uuid2spn");
idlayer.create_uuid2spn(audit)?; idlayer.create_uuid2spn()?;
ltrace!(audit, "Creating index -> uuid2rdn"); ltrace!(audit, "Creating index -> uuid2rdn");
idlayer.create_uuid2rdn(audit)?; idlayer.create_uuid2rdn()?;
self.idxmeta self.idxmeta
.idxkeys .idxkeys
.keys() .keys()
.try_for_each(|ikey| idlayer.create_idx(audit, &ikey.attr, &ikey.itype)) .try_for_each(|ikey| idlayer.create_idx(&ikey.attr, &ikey.itype))
} }
pub fn upgrade_reindex(&self, audit: &mut AuditScope, v: i64) -> Result<(), OperationError> { pub fn upgrade_reindex(&self, audit: &mut AuditScope, v: i64) -> Result<(), OperationError> {
@ -1397,7 +1369,7 @@ impl<'a> BackendWriteTransaction<'a> {
pub fn reindex(&self, audit: &mut AuditScope) -> Result<(), OperationError> { pub fn reindex(&self, audit: &mut AuditScope) -> Result<(), OperationError> {
let idlayer = self.get_idlayer(); let idlayer = self.get_idlayer();
// Purge the idxs // Purge the idxs
unsafe { idlayer.purge_idxs(audit)? }; unsafe { idlayer.purge_idxs()? };
// Using the index metadata on the txn, create all our idx tables // Using the index metadata on the txn, create all our idx tables
self.create_idxs(audit)?; self.create_idxs(audit)?;
@ -1406,7 +1378,7 @@ impl<'a> BackendWriteTransaction<'a> {
// Future idea: Do this in batches of X amount to limit memory // Future idea: Do this in batches of X amount to limit memory
// consumption. // consumption.
let idl = IdList::AllIds; let idl = IdList::AllIds;
let entries = idlayer.get_identry(audit, &idl).map_err(|e| { let entries = idlayer.get_identry(&idl).map_err(|e| {
ladmin_error!(audit, "get_identry failure {:?}", e); ladmin_error!(audit, "get_identry failure {:?}", e);
e e
})?; })?;
@ -1430,10 +1402,10 @@ impl<'a> BackendWriteTransaction<'a> {
})?; })?;
limmediate_warning!(audit, " reindexed {} entries ✅\n", count); limmediate_warning!(audit, " reindexed {} entries ✅\n", count);
limmediate_warning!(audit, "Optimising Indexes ... "); limmediate_warning!(audit, "Optimising Indexes ... ");
idlayer.optimise_dirty_idls(audit); idlayer.optimise_dirty_idls();
limmediate_warning!(audit, "done ✅\n"); limmediate_warning!(audit, "done ✅\n");
limmediate_warning!(audit, "Calculating Index Optimisation Slopes ... "); limmediate_warning!(audit, "Calculating Index Optimisation Slopes ... ");
idlayer.analyse_idx_slopes(audit).map_err(|e| { idlayer.analyse_idx_slopes().map_err(|e| {
ladmin_error!(audit, "index optimisation failed -> {:?}", e); ladmin_error!(audit, "index optimisation failed -> {:?}", e);
e e
})?; })?;
@ -1442,23 +1414,22 @@ impl<'a> BackendWriteTransaction<'a> {
} }
#[cfg(test)] #[cfg(test)]
pub fn purge_idxs(&self, audit: &mut AuditScope) -> Result<(), OperationError> { pub fn purge_idxs(&self) -> Result<(), OperationError> {
unsafe { self.get_idlayer().purge_idxs(audit) } unsafe { self.get_idlayer().purge_idxs() }
} }
#[cfg(test)] #[cfg(test)]
pub fn load_test_idl( pub fn load_test_idl(
&self, &self,
audit: &mut AuditScope,
attr: &String, attr: &String,
itype: &IndexType, itype: &IndexType,
idx_key: &String, idx_key: &String,
) -> Result<Option<IDLBitRange>, OperationError> { ) -> Result<Option<IDLBitRange>, OperationError> {
self.get_idlayer().get_idl(audit, attr, itype, idx_key) self.get_idlayer().get_idl(attr, itype, idx_key)
} }
fn is_idx_slopeyness_generated(&self, audit: &mut AuditScope) -> Result<bool, OperationError> { fn is_idx_slopeyness_generated(&self) -> Result<bool, OperationError> {
self.get_idlayer().is_idx_slopeyness_generated(audit) self.get_idlayer().is_idx_slopeyness_generated()
} }
fn get_idx_slope( fn get_idx_slope(
@ -1469,7 +1440,7 @@ impl<'a> BackendWriteTransaction<'a> {
// Do we have the slopeyness? // Do we have the slopeyness?
let slope = self let slope = self
.get_idlayer() .get_idlayer()
.get_idx_slope(audit, ikey)? .get_idx_slope(ikey)?
.unwrap_or_else(|| get_idx_slope_default(ikey)); .unwrap_or_else(|| get_idx_slope_default(ikey));
ltrace!(audit, "index slope - {:?} -> {:?}", ikey, slope); ltrace!(audit, "index slope - {:?} -> {:?}", ikey, slope);
Ok(slope) Ok(slope)
@ -1484,7 +1455,7 @@ impl<'a> BackendWriteTransaction<'a> {
OperationError::FsError OperationError::FsError
})?; })?;
unsafe { idlayer.purge_id2entry(audit) }.map_err(|e| { unsafe { idlayer.purge_id2entry() }.map_err(|e| {
ladmin_error!(audit, "purge_id2entry failed {:?}", e); ladmin_error!(audit, "purge_id2entry failed {:?}", e);
e e
})?; })?;
@ -1508,12 +1479,12 @@ impl<'a> BackendWriteTransaction<'a> {
}) })
.collect(); .collect();
idlayer.write_identries_raw(audit, identries?.into_iter())?; idlayer.write_identries_raw(identries?.into_iter())?;
// Reindex now we are loaded. // Reindex now we are loaded.
self.reindex(audit)?; self.reindex(audit)?;
let vr = self.verify(audit); let vr = self.verify();
if vr.is_empty() { if vr.is_empty() {
Ok(()) Ok(())
} else { } else {
@ -1521,7 +1492,7 @@ impl<'a> BackendWriteTransaction<'a> {
} }
} }
pub fn commit(self, audit: &mut AuditScope) -> Result<(), OperationError> { pub fn commit(self, _audit: &mut AuditScope) -> Result<(), OperationError> {
let BackendWriteTransaction { let BackendWriteTransaction {
idlayer, idlayer,
idxmeta: _, idxmeta: _,
@ -1531,7 +1502,7 @@ impl<'a> BackendWriteTransaction<'a> {
// Unwrap the Cell we have finished with it. // Unwrap the Cell we have finished with it.
let idlayer = idlayer.into_inner(); let idlayer = idlayer.into_inner();
idlayer.commit(audit).map(|()| { idlayer.commit().map(|()| {
idxmeta_wr.commit(); idxmeta_wr.commit();
}) })
} }
@ -1644,7 +1615,7 @@ impl Backend {
// this has a ::memory() type, but will path == "" work? // this has a ::memory() type, but will path == "" work?
lperf_trace_segment!(audit, "be::new", || { lperf_trace_segment!(audit, "be::new", || {
let idlayer = Arc::new(IdlArcSqlite::new(audit, &cfg, vacuum)?); let idlayer = Arc::new(IdlArcSqlite::new(&cfg, vacuum)?);
let be = Backend { let be = Backend {
cfg, cfg,
idlayer, idlayer,
@ -1657,7 +1628,7 @@ impl Backend {
// the indexing subsystem here. // the indexing subsystem here.
let r = { let r = {
let mut idl_write = be.idlayer.write(); let mut idl_write = be.idlayer.write();
idl_write.setup(audit).and_then(|_| idl_write.commit(audit)) idl_write.setup().and_then(|_| idl_write.commit())
}; };
ltrace!(audit, "be new setup: {:?}", r); ltrace!(audit, "be new setup: {:?}", r);
@ -1731,14 +1702,7 @@ mod tests {
macro_rules! run_test { macro_rules! run_test {
($test_fn:expr) => {{ ($test_fn:expr) => {{
use env_logger; let _ = crate::tracing_tree::test_init();
::std::env::set_var("RUST_LOG", "kanidm=debug");
let _ = env_logger::builder()
.format_timestamp(None)
.format_level(false)
.is_test(true)
.try_init();
let mut audit = AuditScope::new("run_test", uuid::Uuid::new_v4(), None); let mut audit = AuditScope::new("run_test", uuid::Uuid::new_v4(), None);
// This is a demo idxmeta, purely for testing. // This is a demo idxmeta, purely for testing.
@ -1818,9 +1782,9 @@ mod tests {
} }
macro_rules! idl_state { macro_rules! idl_state {
($audit:expr, $be:expr, $attr:expr, $itype:expr, $idx_key:expr, $expect:expr) => {{ ($be:expr, $attr:expr, $itype:expr, $idx_key:expr, $expect:expr) => {{
let t_idl = $be let t_idl = $be
.load_test_idl($audit, &$attr.to_string(), &$itype, &$idx_key.to_string()) .load_test_idl(&$attr.to_string(), &$itype, &$idx_key.to_string())
.expect("IdList Load failed"); .expect("IdList Load failed");
let t = $expect.map(|v: Vec<u64>| IDLBitRange::from_iter(v)); let t = $expect.map(|v: Vec<u64>| IDLBitRange::from_iter(v));
assert_eq!(t_idl, t); assert_eq!(t_idl, t);
@ -2071,7 +2035,7 @@ mod tests {
be.restore(audit, DB_BACKUP_FILE_NAME) be.restore(audit, DB_BACKUP_FILE_NAME)
.expect("Restore failed!"); .expect("Restore failed!");
assert!(be.verify(audit).len() == 0); assert!(be.verify().len() == 0);
}); });
} }
@ -2130,7 +2094,7 @@ mod tests {
be.restore(audit, DB_BACKUP2_FILE_NAME) be.restore(audit, DB_BACKUP2_FILE_NAME)
.expect("Restore failed!"); .expect("Restore failed!");
assert!(be.verify(audit).len() == 0); assert!(be.verify().len() == 0);
}); });
} }
@ -2179,7 +2143,7 @@ mod tests {
be.create(audit, vec![e1.clone(), e2.clone()]).unwrap(); be.create(audit, vec![e1.clone(), e2.clone()]).unwrap();
// purge indexes // purge indexes
be.purge_idxs(audit).unwrap(); be.purge_idxs().unwrap();
// Check they are gone // Check they are gone
let missing = be.missing_idxs(audit).unwrap(); let missing = be.missing_idxs(audit).unwrap();
assert!(missing.len() == 7); assert!(missing.len() == 7);
@ -2189,35 +2153,13 @@ mod tests {
assert!(missing.is_empty()); assert!(missing.is_empty());
// check name and uuid ids on eq, sub, pres // check name and uuid ids on eq, sub, pres
idl_state!( idl_state!(be, "name", IndexType::Equality, "william", Some(vec![1]));
audit,
be, idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![2]));
"name",
IndexType::Equality, idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1, 2]));
"william",
Some(vec![1])
);
idl_state!( idl_state!(
audit,
be,
"name",
IndexType::Equality,
"claire",
Some(vec![2])
);
idl_state!(
audit,
be,
"name",
IndexType::Presence,
"_",
Some(vec![1, 2])
);
idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
@ -2226,7 +2168,6 @@ mod tests {
); );
idl_state!( idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
@ -2234,19 +2175,11 @@ mod tests {
Some(vec![2]) Some(vec![2])
); );
idl_state!( idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![1, 2]));
audit,
be,
"uuid",
IndexType::Presence,
"_",
Some(vec![1, 2])
);
// Show what happens with empty // Show what happens with empty
idl_state!( idl_state!(
audit,
be, be,
"name", "name",
IndexType::Equality, IndexType::Equality,
@ -2255,7 +2188,6 @@ mod tests {
); );
idl_state!( idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
@ -2265,7 +2197,6 @@ mod tests {
let uuid_p_idl = be let uuid_p_idl = be
.load_test_idl( .load_test_idl(
audit,
&"not_indexed".to_string(), &"not_indexed".to_string(),
&IndexType::Presence, &IndexType::Presence,
&"_".to_string(), &"_".to_string(),
@ -2277,15 +2208,15 @@ mod tests {
let claire_uuid = Uuid::parse_str("bd651620-00dd-426b-aaa0-4494f7b7906f").unwrap(); let claire_uuid = Uuid::parse_str("bd651620-00dd-426b-aaa0-4494f7b7906f").unwrap();
let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap(); let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap();
assert!(be.name2uuid(audit, "claire") == Ok(Some(claire_uuid))); assert!(be.name2uuid("claire") == Ok(Some(claire_uuid)));
assert!(be.name2uuid(audit, "william") == Ok(Some(william_uuid))); assert!(be.name2uuid("william") == Ok(Some(william_uuid)));
assert!(be.name2uuid(audit, "db237e8a-0079-4b8c-8a56-593b22aa44d1") == Ok(None)); assert!(be.name2uuid("db237e8a-0079-4b8c-8a56-593b22aa44d1") == Ok(None));
// check uuid2spn // check uuid2spn
assert!(be.uuid2spn(audit, &claire_uuid) == Ok(Some(Value::new_iname("claire")))); assert!(be.uuid2spn(&claire_uuid) == Ok(Some(Value::new_iname("claire"))));
assert!(be.uuid2spn(audit, &william_uuid) == Ok(Some(Value::new_iname("william")))); assert!(be.uuid2spn(&william_uuid) == Ok(Some(Value::new_iname("william"))));
// check uuid2rdn // check uuid2rdn
assert!(be.uuid2rdn(audit, &claire_uuid) == Ok(Some("name=claire".to_string()))); assert!(be.uuid2rdn(&claire_uuid) == Ok(Some("name=claire".to_string())));
assert!(be.uuid2rdn(audit, &william_uuid) == Ok(Some("name=william".to_string()))); assert!(be.uuid2rdn(&william_uuid) == Ok(Some("name=william".to_string())));
}); });
} }
@ -2303,19 +2234,11 @@ mod tests {
let rset = be.create(audit, vec![e1.clone()]).unwrap(); let rset = be.create(audit, vec![e1.clone()]).unwrap();
idl_state!( idl_state!(be, "name", IndexType::Equality, "william", Some(vec![1]));
audit,
be,
"name",
IndexType::Equality,
"william",
Some(vec![1])
);
idl_state!(audit, be, "name", IndexType::Presence, "_", Some(vec![1])); idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1]));
idl_state!( idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
@ -2323,36 +2246,21 @@ mod tests {
Some(vec![1]) Some(vec![1])
); );
idl_state!(audit, be, "uuid", IndexType::Presence, "_", Some(vec![1])); idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![1]));
let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap(); let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap();
assert!(be.name2uuid(audit, "william") == Ok(Some(william_uuid))); assert!(be.name2uuid("william") == Ok(Some(william_uuid)));
assert!(be.uuid2spn(audit, &william_uuid) == Ok(Some(Value::from("william")))); assert!(be.uuid2spn(&william_uuid) == Ok(Some(Value::from("william"))));
assert!(be.uuid2rdn(audit, &william_uuid) == Ok(Some("name=william".to_string()))); assert!(be.uuid2rdn(&william_uuid) == Ok(Some("name=william".to_string())));
// == Now we delete, and assert we removed the items. // == Now we delete, and assert we removed the items.
be.delete(audit, &rset).unwrap(); be.delete(audit, &rset).unwrap();
idl_state!( idl_state!(be, "name", IndexType::Equality, "william", Some(Vec::new()));
audit,
be, idl_state!(be, "name", IndexType::Presence, "_", Some(Vec::new()));
"name",
IndexType::Equality,
"william",
Some(Vec::new())
);
idl_state!( idl_state!(
audit,
be,
"name",
IndexType::Presence,
"_",
Some(Vec::new())
);
idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
@ -2360,18 +2268,11 @@ mod tests {
Some(Vec::new()) Some(Vec::new())
); );
idl_state!( idl_state!(be, "uuid", IndexType::Presence, "_", Some(Vec::new()));
audit,
be,
"uuid",
IndexType::Presence,
"_",
Some(Vec::new())
);
assert!(be.name2uuid(audit, "william") == Ok(None)); assert!(be.name2uuid("william") == Ok(None));
assert!(be.uuid2spn(audit, &william_uuid) == Ok(None)); assert!(be.uuid2spn(&william_uuid) == Ok(None));
assert!(be.uuid2rdn(audit, &william_uuid) == Ok(None)); assert!(be.uuid2rdn(&william_uuid) == Ok(None));
}) })
} }
@ -2406,19 +2307,11 @@ mod tests {
// Now remove e1, e3. // Now remove e1, e3.
be.delete(audit, &rset).unwrap(); be.delete(audit, &rset).unwrap();
idl_state!( idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![2]));
audit,
be,
"name",
IndexType::Equality,
"claire",
Some(vec![2])
);
idl_state!(audit, be, "name", IndexType::Presence, "_", Some(vec![2])); idl_state!(be, "name", IndexType::Presence, "_", Some(vec![2]));
idl_state!( idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
@ -2426,23 +2319,23 @@ mod tests {
Some(vec![2]) Some(vec![2])
); );
idl_state!(audit, be, "uuid", IndexType::Presence, "_", Some(vec![2])); idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![2]));
let claire_uuid = Uuid::parse_str("bd651620-00dd-426b-aaa0-4494f7b7906f").unwrap(); let claire_uuid = Uuid::parse_str("bd651620-00dd-426b-aaa0-4494f7b7906f").unwrap();
let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap(); let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap();
let lucy_uuid = Uuid::parse_str("7b23c99d-c06b-4a9a-a958-3afa56383e1d").unwrap(); let lucy_uuid = Uuid::parse_str("7b23c99d-c06b-4a9a-a958-3afa56383e1d").unwrap();
assert!(be.name2uuid(audit, "claire") == Ok(Some(claire_uuid))); assert!(be.name2uuid("claire") == Ok(Some(claire_uuid)));
assert!(be.uuid2spn(audit, &claire_uuid) == Ok(Some(Value::from("claire")))); assert!(be.uuid2spn(&claire_uuid) == Ok(Some(Value::from("claire"))));
assert!(be.uuid2rdn(audit, &claire_uuid) == Ok(Some("name=claire".to_string()))); assert!(be.uuid2rdn(&claire_uuid) == Ok(Some("name=claire".to_string())));
assert!(be.name2uuid(audit, "william") == Ok(None)); assert!(be.name2uuid("william") == Ok(None));
assert!(be.uuid2spn(audit, &william_uuid) == Ok(None)); assert!(be.uuid2spn(&william_uuid) == Ok(None));
assert!(be.uuid2rdn(audit, &william_uuid) == Ok(None)); assert!(be.uuid2rdn(&william_uuid) == Ok(None));
assert!(be.name2uuid(audit, "lucy") == Ok(None)); assert!(be.name2uuid("lucy") == Ok(None));
assert!(be.uuid2spn(audit, &lucy_uuid) == Ok(None)); assert!(be.uuid2spn(&lucy_uuid) == Ok(None));
assert!(be.uuid2rdn(audit, &lucy_uuid) == Ok(None)); assert!(be.uuid2rdn(&lucy_uuid) == Ok(None));
}) })
} }
@ -2475,27 +2368,20 @@ mod tests {
be.modify(audit, &rset, &vec![ce1]).unwrap(); be.modify(audit, &rset, &vec![ce1]).unwrap();
// Now check the idls // Now check the idls
idl_state!( idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![1]));
audit,
be,
"name",
IndexType::Equality,
"claire",
Some(vec![1])
);
idl_state!(audit, be, "name", IndexType::Presence, "_", Some(vec![1])); idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1]));
idl_state!(audit, be, "tb", IndexType::Equality, "test", Some(vec![1])); idl_state!(be, "tb", IndexType::Equality, "test", Some(vec![1]));
idl_state!(audit, be, "ta", IndexType::Equality, "test", Some(vec![])); idl_state!(be, "ta", IndexType::Equality, "test", Some(vec![]));
// let claire_uuid = Uuid::parse_str("bd651620-00dd-426b-aaa0-4494f7b7906f").unwrap(); // let claire_uuid = Uuid::parse_str("bd651620-00dd-426b-aaa0-4494f7b7906f").unwrap();
let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap(); let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap();
assert!(be.name2uuid(audit, "william") == Ok(None)); assert!(be.name2uuid("william") == Ok(None));
assert!(be.name2uuid(audit, "claire") == Ok(Some(william_uuid))); assert!(be.name2uuid("claire") == Ok(Some(william_uuid)));
assert!(be.uuid2spn(audit, &william_uuid) == Ok(Some(Value::from("claire")))); assert!(be.uuid2spn(&william_uuid) == Ok(Some(Value::from("claire"))));
assert!(be.uuid2rdn(audit, &william_uuid) == Ok(Some("name=claire".to_string()))); assert!(be.uuid2rdn(&william_uuid) == Ok(Some("name=claire".to_string())));
}) })
} }
@ -2522,17 +2408,9 @@ mod tests {
be.modify(audit, &rset, &vec![ce1]).unwrap(); be.modify(audit, &rset, &vec![ce1]).unwrap();
idl_state!( idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![1]));
audit,
be,
"name",
IndexType::Equality,
"claire",
Some(vec![1])
);
idl_state!( idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
@ -2540,34 +2418,26 @@ mod tests {
Some(vec![1]) Some(vec![1])
); );
idl_state!(audit, be, "name", IndexType::Presence, "_", Some(vec![1])); idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1]));
idl_state!(audit, be, "uuid", IndexType::Presence, "_", Some(vec![1])); idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![1]));
idl_state!( idl_state!(
audit,
be, be,
"uuid", "uuid",
IndexType::Equality, IndexType::Equality,
"db237e8a-0079-4b8c-8a56-593b22aa44d1", "db237e8a-0079-4b8c-8a56-593b22aa44d1",
Some(Vec::new()) Some(Vec::new())
); );
idl_state!( idl_state!(be, "name", IndexType::Equality, "william", Some(Vec::new()));
audit,
be,
"name",
IndexType::Equality,
"william",
Some(Vec::new())
);
let claire_uuid = Uuid::parse_str("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8").unwrap(); let claire_uuid = Uuid::parse_str("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8").unwrap();
let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap(); let william_uuid = Uuid::parse_str("db237e8a-0079-4b8c-8a56-593b22aa44d1").unwrap();
assert!(be.name2uuid(audit, "william") == Ok(None)); assert!(be.name2uuid("william") == Ok(None));
assert!(be.name2uuid(audit, "claire") == Ok(Some(claire_uuid))); assert!(be.name2uuid("claire") == Ok(Some(claire_uuid)));
assert!(be.uuid2spn(audit, &william_uuid) == Ok(None)); assert!(be.uuid2spn(&william_uuid) == Ok(None));
assert!(be.uuid2rdn(audit, &william_uuid) == Ok(None)); assert!(be.uuid2rdn(&william_uuid) == Ok(None));
assert!(be.uuid2spn(audit, &claire_uuid) == Ok(Some(Value::from("claire")))); assert!(be.uuid2spn(&claire_uuid) == Ok(Some(Value::from("claire"))));
assert!(be.uuid2rdn(audit, &claire_uuid) == Ok(Some("name=claire".to_string()))); assert!(be.uuid2rdn(&claire_uuid) == Ok(Some("name=claire".to_string())));
}) })
} }
@ -2867,7 +2737,7 @@ mod tests {
run_test!(|audit: &mut AuditScope, be: &mut BackendWriteTransaction| { run_test!(|audit: &mut AuditScope, be: &mut BackendWriteTransaction| {
// Test where the index is in schema but not created (purge idxs) // Test where the index is in schema but not created (purge idxs)
// should fall back to an empty set because we can't satisfy the term // should fall back to an empty set because we can't satisfy the term
be.purge_idxs(audit).unwrap(); be.purge_idxs().unwrap();
debug!("{:?}", be.missing_idxs(audit).unwrap()); debug!("{:?}", be.missing_idxs(audit).unwrap());
let f_eq = let f_eq =
unsafe { filter_resolved!(f_eq("name", PartialValue::new_utf8s("william"))) }; unsafe { filter_resolved!(f_eq("name", PartialValue::new_utf8s("william"))) };
@ -2913,7 +2783,7 @@ mod tests {
// If the slopes haven't been generated yet, there are some hardcoded values // If the slopes haven't been generated yet, there are some hardcoded values
// that we can use instead. They aren't generated until a first re-index. // that we can use instead. They aren't generated until a first re-index.
assert!(!be.is_idx_slopeyness_generated(audit).unwrap()); assert!(!be.is_idx_slopeyness_generated().unwrap());
let ta_eq_slope = be let ta_eq_slope = be
.get_idx_slope(audit, &IdxKey::new("ta", IndexType::Equality)) .get_idx_slope(audit, &IdxKey::new("ta", IndexType::Equality))
@ -2947,7 +2817,7 @@ mod tests {
// Now check slope generation for the values. Today these are calculated // Now check slope generation for the values. Today these are calculated
// at reindex time, so we now perform the re-index. // at reindex time, so we now perform the re-index.
assert!(be.reindex(audit).is_ok()); assert!(be.reindex(audit).is_ok());
assert!(be.is_idx_slopeyness_generated(audit).unwrap()); assert!(be.is_idx_slopeyness_generated().unwrap());
let ta_eq_slope = be let ta_eq_slope = be
.get_idx_slope(audit, &IdxKey::new("ta", IndexType::Equality)) .get_idx_slope(audit, &IdxKey::new("ta", IndexType::Equality))

8
kanidmd/src/lib/core/mod.rs
View file

@ -142,7 +142,7 @@ pub fn dbscan_list_indexes_core(config: &Configuration) {
let be = dbscan_setup_be!(audit, &config); let be = dbscan_setup_be!(audit, &config);
let be_rotxn = be.read(); let be_rotxn = be.read();
match be_rotxn.list_indexes(&mut audit) { match be_rotxn.list_indexes() {
Ok(mut idx_list) => { Ok(mut idx_list) => {
idx_list.sort_unstable(); idx_list.sort_unstable();
idx_list.iter().for_each(|idx_name| { idx_list.iter().for_each(|idx_name| {
@ -165,7 +165,7 @@ pub fn dbscan_list_id2entry_core(config: &Configuration) {
let be = dbscan_setup_be!(audit, &config); let be = dbscan_setup_be!(audit, &config);
let be_rotxn = be.read(); let be_rotxn = be.read();
match be_rotxn.list_id2entry(&mut audit) { match be_rotxn.list_id2entry() {
Ok(mut id_list) => { Ok(mut id_list) => {
id_list.sort_unstable_by_key(|k| k.0); id_list.sort_unstable_by_key(|k| k.0);
id_list.iter().for_each(|(id, value)| { id_list.iter().for_each(|(id, value)| {
@ -195,7 +195,7 @@ pub fn dbscan_list_index_core(config: &Configuration, index_name: &str) {
let be = dbscan_setup_be!(audit, &config); let be = dbscan_setup_be!(audit, &config);
let be_rotxn = be.read(); let be_rotxn = be.read();
match be_rotxn.list_index_content(&mut audit, index_name) { match be_rotxn.list_index_content(index_name) {
Ok(mut idx_list) => { Ok(mut idx_list) => {
idx_list.sort_unstable_by(|a, b| a.0.cmp(&b.0)); idx_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
idx_list.iter().for_each(|(key, value)| { idx_list.iter().for_each(|(key, value)| {
@ -218,7 +218,7 @@ pub fn dbscan_get_id2entry_core(config: &Configuration, id: u64) {
let be = dbscan_setup_be!(audit, &config); let be = dbscan_setup_be!(audit, &config);
let be_rotxn = be.read(); let be_rotxn = be.read();
match be_rotxn.get_id2entry(&mut audit, id) { match be_rotxn.get_id2entry(id) {
Ok((id, value)) => println!("{:>8}: {}", id, value), Ok((id, value)) => println!("{:>8}: {}", id, value),
Err(e) => { Err(e) => {
audit.write_log(); audit.write_log();

14
kanidmd/src/lib/entry.rs
View file

@ -1294,9 +1294,7 @@ impl Entry<EntrySealed, EntryCommitted> {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
// Why is this returning a `Result<Self, ()>` when we could just do `Option<Self>` pub fn from_dbentry(db_e: DbEntry, id: u64) -> Option<Self> {
// How did this even pass clippy?
pub fn from_dbentry(au: &mut AuditScope, db_e: DbEntry, id: u64) -> Result<Self, ()> {
// Convert attrs from db format to value // Convert attrs from db format to value
let r_attrs: Result<Map<AttrString, ValueSet>, ()> = match db_e.ent { let r_attrs: Result<Map<AttrString, ValueSet>, ()> = match db_e.ent {
DbEntryVers::V1(v1) => v1 DbEntryVers::V1(v1) => v1
@ -1309,7 +1307,6 @@ impl Entry<EntrySealed, EntryCommitted> {
Ok(vv) => Ok((k, vv)), Ok(vv) => Ok((k, vv)),
Err(()) => { Err(()) => {
admin_error!(value = ?k, "from_dbentry failed"); admin_error!(value = ?k, "from_dbentry failed");
ladmin_error!(au, "from_dbentry failed on value {:?}", k);
Err(()) Err(())
} }
} }
@ -1317,18 +1314,15 @@ impl Entry<EntrySealed, EntryCommitted> {
.collect(), .collect(),
}; };
let attrs = r_attrs?; let attrs = r_attrs.ok()?;
let uuid: Uuid = *match attrs.get("uuid") { let uuid: Uuid = *match attrs.get("uuid") {
Some(vs) => vs.iter().take(1).next(), Some(vs) => vs.iter().take(1).next(),
None => None, None => None,
} }
.ok_or(())? .and_then(|v| v.to_uuid())?;
// Now map value -> uuid
.to_uuid()
.ok_or(())?;
Ok(Entry { Some(Entry {
valid: EntrySealed { uuid }, valid: EntrySealed { uuid },
state: EntryCommitted { id }, state: EntryCommitted { id },
attrs, attrs,

14
kanidmd/src/lib/server.rs
View file

@ -286,12 +286,12 @@ pub trait QueryServerTransaction<'a> {
// Remember, we don't care if the name is invalid, because search // Remember, we don't care if the name is invalid, because search
// will validate/normalise the filter we construct for us. COOL! // will validate/normalise the filter we construct for us. COOL!
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn name_to_uuid(&self, audit: &mut AuditScope, name: &str) -> Result<Uuid, OperationError> { fn name_to_uuid(&self, _audit: &mut AuditScope, name: &str) -> Result<Uuid, OperationError> {
// Is it just a uuid? // Is it just a uuid?
Uuid::parse_str(name).or_else(|_| { Uuid::parse_str(name).or_else(|_| {
let lname = name.to_lowercase(); let lname = name.to_lowercase();
self.get_be_txn() self.get_be_txn()
.name2uuid(audit, lname.as_str())? .name2uuid(lname.as_str())?
.ok_or(OperationError::NoMatchingEntries) // should we log this? .ok_or(OperationError::NoMatchingEntries) // should we log this?
}) })
} }
@ -299,10 +299,10 @@ pub trait QueryServerTransaction<'a> {
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn uuid_to_spn( fn uuid_to_spn(
&self, &self,
audit: &mut AuditScope, _audit: &mut AuditScope,
uuid: &Uuid, uuid: &Uuid,
) -> Result<Option<Value>, OperationError> { ) -> Result<Option<Value>, OperationError> {
let r = self.get_be_txn().uuid2spn(audit, uuid)?; let r = self.get_be_txn().uuid2spn(uuid)?;
if let Some(ref n) = r { if let Some(ref n) = r {
// Shouldn't we be doing more graceful error handling here? // Shouldn't we be doing more graceful error handling here?
@ -314,10 +314,10 @@ pub trait QueryServerTransaction<'a> {
} }
// ! TRACING INTEGRATED // ! TRACING INTEGRATED
fn uuid_to_rdn(&self, audit: &mut AuditScope, uuid: &Uuid) -> Result<String, OperationError> { fn uuid_to_rdn(&self, _audit: &mut AuditScope, uuid: &Uuid) -> Result<String, OperationError> {
// If we have a some, pass it on, else unwrap into a default. // If we have a some, pass it on, else unwrap into a default.
self.get_be_txn() self.get_be_txn()
.uuid2rdn(audit, uuid) .uuid2rdn(uuid)
.map(|v| v.unwrap_or_else(|| format!("uuid={}", uuid.to_hyphenated_ref()))) .map(|v| v.unwrap_or_else(|| format!("uuid={}", uuid.to_hyphenated_ref())))
} }
@ -844,7 +844,7 @@ impl<'a> QueryServerReadTransaction<'a> {
// If we fail after backend, we need to return NOW because we can't // If we fail after backend, we need to return NOW because we can't
// assert any other faith in the DB states. // assert any other faith in the DB states.
// * backend // * backend
let be_errs = self.get_be_txn().verify(audit); let be_errs = self.get_be_txn().verify();
if !be_errs.is_empty() { if !be_errs.is_empty() {
return be_errs; return be_errs;