mart-w/kanidm
1
0
Fork 0
mirror of https://github.com/kanidm/kanidm.git synced 2025-02-23 20:47:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refint improve (#274)

Browse source 
Fixes #61 and fixes #234 - this rewrites quite a few internals of refint and memberof to make them much more efficient compared to previously. This takes nearly 70s out of the test execution time - a full 25% of the run time of tests.

A number of other improvements have been made through out with regard to memory pre-alloc for hashset/hashmap, fixing some more types, and reducing some un-needed allocations.
This commit is contained in:
Firstyear 2020-06-26 11:36:37 +10:00 committed by GitHub
parent 0adec0d437
commit 96e6c9107d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 952 additions and 614 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
Cargo.lock generated
View file

@ -354,6 +354,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d"
[[package]]
name = "ahash"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.7.10" version = "0.7.10"
@ -1294,6 +1300,16 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
[[package]]
name = "hashbrown"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab9b7860757ce258c89fd48d28b68c41713e597a7b09e793f6c6a6e2ea37c827"
dependencies = [
"ahash",
"autocfg",
]
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.3.1" version = "0.3.1"
@ -1518,6 +1534,7 @@ dependencies = [
"env_logger", "env_logger",
"futures", "futures",
"futures-util", "futures-util",
"hashbrown",
"idlset", "idlset",
"kanidm_proto", "kanidm_proto",
"lazy_static", "lazy_static",

3
kanidm_client/tests/proto_v1_test.rs
View file

@ -763,6 +763,9 @@ fn test_server_rest_totp_auth_lifecycle() {
.unwrap(), .unwrap(),
) )
.expect("Failed to do totp?"); .expect("Failed to do totp?");
// TODO: It's extremely rare, but it's happened ONCE where, the time window
// elapsed DURING this test, so there is a minor possibility of this actually
// having a false negative. Is it possible to prevent this?
assert!(rsclient_good assert!(rsclient_good
.auth_password_totp("demo_account", "sohdi3iuHo6mai7noh0a", totp) .auth_password_totp("demo_account", "sohdi3iuHo6mai7noh0a", totp)
.is_ok()); .is_ok());

2
kanidm_proto/src/v1.rs
View file

@ -293,7 +293,7 @@ impl fmt::Display for Entry {
} }
} }
#[derive(Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, Eq, PartialEq)] #[derive(Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
pub enum Filter { pub enum Filter {
// This is attr - value // This is attr - value

1
kanidmd/Cargo.toml
View file

@ -60,6 +60,7 @@ r2d2_sqlite = "0.14"
structopt = { version = "0.3", default-features = false } structopt = { version = "0.3", default-features = false }
time = "0.1" time = "0.1"
hashbrown = "0.8"
concread = "0.1" concread = "0.1"
# concread = { path = "../../concread" } # concread = { path = "../../concread" }
crossbeam = "0.7" crossbeam = "0.7"

7
kanidmd/src/lib/access.rs
View file

@ -19,9 +19,8 @@
use concread::cowcell::*; use concread::cowcell::*;
use kanidm_proto::v1::Filter as ProtoFilter; use kanidm_proto::v1::Filter as ProtoFilter;
use kanidm_proto::v1::OperationError; use kanidm_proto::v1::OperationError;
// use std::collections::BTreeMap;
use std::collections::BTreeSet; use std::collections::BTreeSet;
// use std::collections::HashSet; // use hashbrown::HashSet;
use std::ops::DerefMut; use std::ops::DerefMut;
use uuid::Uuid; use uuid::Uuid;
@ -355,7 +354,6 @@ impl AccessControlProfile {
#[derive(Clone)] #[derive(Clone)]
struct AccessControlsInner { struct AccessControlsInner {
// acps_search: BTreeMap<Uuid, AccessControlSearch>,
acps_search: Vec<AccessControlSearch>, acps_search: Vec<AccessControlSearch>,
acps_create: Vec<AccessControlCreate>, acps_create: Vec<AccessControlCreate>,
acps_modify: Vec<AccessControlModify>, acps_modify: Vec<AccessControlModify>,
@ -367,7 +365,6 @@ pub struct AccessControls {
} }
pub trait AccessControlsTransaction { pub trait AccessControlsTransaction {
// fn get_search(&self) -> &BTreeMap<Uuid, AccessControlSearch>;
fn get_search(&self) -> &Vec<AccessControlSearch>; fn get_search(&self) -> &Vec<AccessControlSearch>;
fn get_create(&self) -> &Vec<AccessControlCreate>; fn get_create(&self) -> &Vec<AccessControlCreate>;
fn get_modify(&self) -> &Vec<AccessControlModify>; fn get_modify(&self) -> &Vec<AccessControlModify>;
@ -1244,7 +1241,6 @@ impl<'a> AccessControlsWriteTransaction<'a> {
} }
impl<'a> AccessControlsTransaction for AccessControlsWriteTransaction<'a> { impl<'a> AccessControlsTransaction for AccessControlsWriteTransaction<'a> {
// fn get_search(&self) -> &BTreeMap<Uuid, AccessControlSearch> {
fn get_search(&self) -> &Vec<AccessControlSearch> { fn get_search(&self) -> &Vec<AccessControlSearch> {
&self.inner.acps_search &self.inner.acps_search
} }
@ -1271,7 +1267,6 @@ pub struct AccessControlsReadTransaction {
} }
impl AccessControlsTransaction for AccessControlsReadTransaction { impl AccessControlsTransaction for AccessControlsReadTransaction {
// fn get_search(&self) -> &BTreeMap<Uuid, AccessControlSearch> {
fn get_search(&self) -> &Vec<AccessControlSearch> { fn get_search(&self) -> &Vec<AccessControlSearch> {
&self.inner.acps_search &self.inner.acps_search
} }

147
kanidmd/src/lib/be/mod.rs
View file

@ -2,8 +2,8 @@ use std::convert::TryFrom;
use std::fs; use std::fs;
use crate::value::IndexType; use crate::value::IndexType;
use hashbrown::HashSet as Set;
use std::cell::UnsafeCell; use std::cell::UnsafeCell;
use std::collections::HashSet as Set;
use std::sync::Arc; use std::sync::Arc;
use crate::audit::AuditScope; use crate::audit::AuditScope;
@ -102,6 +102,7 @@ pub trait BackendTransaction {
/// Recursively apply a filter, transforming into IDL's on the way. This builds a query /// Recursively apply a filter, transforming into IDL's on the way. This builds a query
/// execution log, so that it can be examined how an operation proceeded. /// execution log, so that it can be examined how an operation proceeded.
#[allow(clippy::cognitive_complexity)]
fn filter2idl( fn filter2idl(
&self, &self,
au: &mut AuditScope, au: &mut AuditScope,
@ -413,6 +414,44 @@ pub trait BackendTransaction {
// debug!("final cand set ==> {:?}", cand_idl); // debug!("final cand set ==> {:?}", cand_idl);
(cand_idl, setplan) (cand_idl, setplan)
} // end and } // end and
FilterResolved::Inclusion(l) => {
// For inclusion to be valid, every term must have *at least* one element present.
// This really relies on indexing, and so it's internal only - generally only
// for fully indexed existance queries, such as from refint.
// This has a lot in common with an And and Or but not really quite either.
let mut plan = Vec::new();
let mut result = IDLBitRange::new();
// For each filter in l
for f in l.iter() {
// get their idls
match self.filter2idl(au, f, thres)? {
(IDL::Indexed(idl), fp) => {
plan.push(fp);
if idl.is_empty() {
// It's empty, so something is missing. Bail fast.
lfilter!(au, "Inclusion is unable to proceed - an empty (missing) item was found!");
let setplan = FilterPlan::InclusionIndexed(plan);
return Ok((IDL::Indexed(IDLBitRange::new()), setplan));
} else {
result = result | idl;
}
}
(_, fp) => {
plan.push(fp);
lfilter_error!(
au,
"Inclusion is unable to proceed - all terms must be fully indexed!"
);
let setplan = FilterPlan::InclusionInvalid(plan);
return Ok((IDL::Partial(IDLBitRange::new()), setplan));
}
}
} // end or.iter()
// If we got here, every term must have been indexed
let setplan = FilterPlan::InclusionIndexed(plan);
(IDL::Indexed(result), setplan)
}
// So why does this return empty? Normally we actually process an AndNot in the context // So why does this return empty? Normally we actually process an AndNot in the context
// of an "AND" query, but if it's used anywhere else IE the root filter, then there is // of an "AND" query, but if it's used anywhere else IE the root filter, then there is
// no other set to exclude - therefore it's empty set. Additionally, even in an OR query // no other set to exclude - therefore it's empty set. Additionally, even in an OR query
@ -1312,8 +1351,8 @@ impl Backend {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use hashbrown::HashSet as Set;
use idlset::IDLBitRange; use idlset::IDLBitRange;
use std::collections::HashSet as Set;
use std::fs; use std::fs;
use std::iter::FromIterator; use std::iter::FromIterator;
use uuid::Uuid; use uuid::Uuid;
@ -1337,7 +1376,7 @@ mod tests {
let mut audit = AuditScope::new("run_test", uuid::Uuid::new_v4(), None); let mut audit = AuditScope::new("run_test", uuid::Uuid::new_v4(), None);
// This is a demo idxmeta, purely for testing. // This is a demo idxmeta, purely for testing.
let mut idxmeta = Set::new(); let mut idxmeta = Set::with_capacity(16);
idxmeta.insert(IdxKey { idxmeta.insert(IdxKey {
attr: "name".to_string(), attr: "name".to_string(),
itype: IndexType::EQUALITY, itype: IndexType::EQUALITY,
@ -1428,8 +1467,8 @@ mod tests {
assert_eq!(empty_result, Err(OperationError::EmptyRequest)); assert_eq!(empty_result, Err(OperationError::EmptyRequest));
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
e.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let e = unsafe { e.into_sealed_new() }; let e = unsafe { e.into_sealed_new() };
let single_result = be.create(audit, vec![e.clone()]); let single_result = be.create(audit, vec![e.clone()]);
@ -1447,8 +1486,8 @@ mod tests {
ltrace!(audit, "Simple Search"); ltrace!(audit, "Simple Search");
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("userid", &Value::from("claire")); e.add_ava("userid", Value::from("claire"));
e.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let e = unsafe { e.into_sealed_new() }; let e = unsafe { e.into_sealed_new() };
let single_result = be.create(audit, vec![e.clone()]); let single_result = be.create(audit, vec![e.clone()]);
@ -1475,12 +1514,12 @@ mod tests {
ltrace!(audit, "Simple Modify"); ltrace!(audit, "Simple Modify");
// First create some entries (3?) // First create some entries (3?)
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("userid", &Value::from("william")); e1.add_ava("userid", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("userid", &Value::from("alice")); e2.add_ava("userid", Value::from("alice"));
e2.add_ava("uuid", &Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e")); e2.add_ava("uuid", Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"));
let ve1 = unsafe { e1.clone().into_sealed_new() }; let ve1 = unsafe { e1.clone().into_sealed_new() };
let ve2 = unsafe { e2.clone().into_sealed_new() }; let ve2 = unsafe { e2.clone().into_sealed_new() };
@ -1512,8 +1551,8 @@ mod tests {
// Make some changes to r1, r2. // Make some changes to r1, r2.
let pre1 = unsafe { r1.clone().into_sealed_committed() }; let pre1 = unsafe { r1.clone().into_sealed_committed() };
let pre2 = unsafe { r2.clone().into_sealed_committed() }; let pre2 = unsafe { r2.clone().into_sealed_committed() };
r1.add_ava("desc", &Value::from("modified")); r1.add_ava("desc", Value::from("modified"));
r2.add_ava("desc", &Value::from("modified")); r2.add_ava("desc", Value::from("modified"));
// Now ... cheat. // Now ... cheat.
@ -1549,16 +1588,16 @@ mod tests {
// First create some entries (3?) // First create some entries (3?)
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("userid", &Value::from("william")); e1.add_ava("userid", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("userid", &Value::from("alice")); e2.add_ava("userid", Value::from("alice"));
e2.add_ava("uuid", &Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e")); e2.add_ava("uuid", Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"));
let mut e3: Entry<EntryInit, EntryNew> = Entry::new(); let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
e3.add_ava("userid", &Value::from("lucy")); e3.add_ava("userid", Value::from("lucy"));
e3.add_ava("uuid", &Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d")); e3.add_ava("uuid", Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"));
let ve1 = unsafe { e1.clone().into_sealed_new() }; let ve1 = unsafe { e1.clone().into_sealed_new() };
let ve2 = unsafe { e2.clone().into_sealed_new() }; let ve2 = unsafe { e2.clone().into_sealed_new() };
@ -1590,8 +1629,8 @@ mod tests {
// WARNING: Normally, this isn't possible, but we are pursposefully breaking // WARNING: Normally, this isn't possible, but we are pursposefully breaking
// the state machine rules here!!!! // the state machine rules here!!!!
let mut e4: Entry<EntryInit, EntryNew> = Entry::new(); let mut e4: Entry<EntryInit, EntryNew> = Entry::new();
e4.add_ava("userid", &Value::from("amy")); e4.add_ava("userid", Value::from("amy"));
e4.add_ava("uuid", &Value::from("21d816b5-1f6a-4696-b7c1-6ed06d22ed81")); e4.add_ava("uuid", Value::from("21d816b5-1f6a-4696-b7c1-6ed06d22ed81"));
let ve4 = unsafe { e4.clone().into_sealed_committed() }; let ve4 = unsafe { e4.clone().into_sealed_committed() };
@ -1619,16 +1658,16 @@ mod tests {
run_test!(|audit: &mut AuditScope, be: &mut BackendWriteTransaction| { run_test!(|audit: &mut AuditScope, be: &mut BackendWriteTransaction| {
// First create some entries (3?) // First create some entries (3?)
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("userid", &Value::from("william")); e1.add_ava("userid", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("userid", &Value::from("alice")); e2.add_ava("userid", Value::from("alice"));
e2.add_ava("uuid", &Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e")); e2.add_ava("uuid", Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"));
let mut e3: Entry<EntryInit, EntryNew> = Entry::new(); let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
e3.add_ava("userid", &Value::from("lucy")); e3.add_ava("userid", Value::from("lucy"));
e3.add_ava("uuid", &Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d")); e3.add_ava("uuid", Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"));
let ve1 = unsafe { e1.clone().into_sealed_new() }; let ve1 = unsafe { e1.clone().into_sealed_new() };
let ve2 = unsafe { e2.clone().into_sealed_new() }; let ve2 = unsafe { e2.clone().into_sealed_new() };
@ -1693,13 +1732,13 @@ mod tests {
run_test!(|audit: &mut AuditScope, be: &mut BackendWriteTransaction| { run_test!(|audit: &mut AuditScope, be: &mut BackendWriteTransaction| {
// Add some test data? // Add some test data?
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("name", &Value::new_iname_s("william")); e1.add_ava("name", Value::new_iname_s("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let e1 = unsafe { e1.into_sealed_new() }; let e1 = unsafe { e1.into_sealed_new() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("name", &Value::new_iname_s("claire")); e2.add_ava("name", Value::new_iname_s("claire"));
e2.add_ava("uuid", &Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f")); e2.add_ava("uuid", Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f"));
let e2 = unsafe { e2.into_sealed_new() }; let e2 = unsafe { e2.into_sealed_new() };
be.create(audit, vec![e1.clone(), e2.clone()]).unwrap(); be.create(audit, vec![e1.clone(), e2.clone()]).unwrap();
@ -1823,8 +1862,8 @@ mod tests {
// Test that on entry create, the indexes are made correctly. // Test that on entry create, the indexes are made correctly.
// this is a similar case to reindex. // this is a similar case to reindex.
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("name", &Value::from("william")); e1.add_ava("name", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let e1 = unsafe { e1.into_sealed_new() }; let e1 = unsafe { e1.into_sealed_new() };
let rset = be.create(audit, vec![e1.clone()]).unwrap(); let rset = be.create(audit, vec![e1.clone()]).unwrap();
@ -1910,18 +1949,18 @@ mod tests {
// Test that on entry create, the indexes are made correctly. // Test that on entry create, the indexes are made correctly.
// this is a similar case to reindex. // this is a similar case to reindex.
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("name", &Value::from("william")); e1.add_ava("name", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let e1 = unsafe { e1.into_sealed_new() }; let e1 = unsafe { e1.into_sealed_new() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("name", &Value::from("claire")); e2.add_ava("name", Value::from("claire"));
e2.add_ava("uuid", &Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f")); e2.add_ava("uuid", Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f"));
let e2 = unsafe { e2.into_sealed_new() }; let e2 = unsafe { e2.into_sealed_new() };
let mut e3: Entry<EntryInit, EntryNew> = Entry::new(); let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
e3.add_ava("userid", &Value::from("lucy")); e3.add_ava("userid", Value::from("lucy"));
e3.add_ava("uuid", &Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d")); e3.add_ava("uuid", Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"));
let e3 = unsafe { e3.into_sealed_new() }; let e3 = unsafe { e3.into_sealed_new() };
let mut rset = be let mut rset = be
@ -1980,21 +2019,21 @@ mod tests {
// us. For the test to be "accurate" we must add one attr, remove one attr // us. For the test to be "accurate" we must add one attr, remove one attr
// and change one attr. // and change one attr.
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("name", &Value::from("william")); e1.add_ava("name", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
e1.add_ava("ta", &Value::from("test")); e1.add_ava("ta", Value::from("test"));
let e1 = unsafe { e1.into_sealed_new() }; let e1 = unsafe { e1.into_sealed_new() };
let rset = be.create(audit, vec![e1.clone()]).unwrap(); let rset = be.create(audit, vec![e1.clone()]).unwrap();
// Now, alter the new entry. // Now, alter the new entry.
let mut ce1 = unsafe { rset[0].clone().into_invalid() }; let mut ce1 = unsafe { rset[0].clone().into_invalid() };
// add something. // add something.
ce1.add_ava("tb", &Value::from("test")); ce1.add_ava("tb", Value::from("test"));
// remove something. // remove something.
ce1.purge_ava("ta"); ce1.purge_ava("ta");
// mod something. // mod something.
ce1.purge_ava("name"); ce1.purge_ava("name");
ce1.add_ava("name", &Value::from("claire")); ce1.add_ava("name", Value::from("claire"));
let ce1 = unsafe { ce1.into_sealed_committed() }; let ce1 = unsafe { ce1.into_sealed_committed() };
@ -2033,8 +2072,8 @@ mod tests {
// This will be needing to be correct for conflicts when we add // This will be needing to be correct for conflicts when we add
// replication support! // replication support!
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("name", &Value::from("william")); e1.add_ava("name", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
let e1 = unsafe { e1.into_sealed_new() }; let e1 = unsafe { e1.into_sealed_new() };
let rset = be.create(audit, vec![e1.clone()]).unwrap(); let rset = be.create(audit, vec![e1.clone()]).unwrap();
@ -2042,8 +2081,8 @@ mod tests {
let mut ce1 = unsafe { rset[0].clone().into_invalid() }; let mut ce1 = unsafe { rset[0].clone().into_invalid() };
ce1.purge_ava("name"); ce1.purge_ava("name");
ce1.purge_ava("uuid"); ce1.purge_ava("uuid");
ce1.add_ava("name", &Value::from("claire")); ce1.add_ava("name", Value::from("claire"));
ce1.add_ava("uuid", &Value::from("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8")); ce1.add_ava("uuid", Value::from("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8"));
let ce1 = unsafe { ce1.into_sealed_committed() }; let ce1 = unsafe { ce1.into_sealed_committed() };
be.modify(audit, &rset, &vec![ce1]).unwrap(); be.modify(audit, &rset, &vec![ce1]).unwrap();
@ -2104,15 +2143,15 @@ mod tests {
// Create a test entry with some indexed / unindexed values. // Create a test entry with some indexed / unindexed values.
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("name", &Value::from("william")); e1.add_ava("name", Value::from("william"));
e1.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1")); e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
e1.add_ava("no-index", &Value::from("william")); e1.add_ava("no-index", Value::from("william"));
e1.add_ava("other-no-index", &Value::from("william")); e1.add_ava("other-no-index", Value::from("william"));
let e1 = unsafe { e1.into_sealed_new() }; let e1 = unsafe { e1.into_sealed_new() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("name", &Value::from("claire")); e2.add_ava("name", Value::from("claire"));
e2.add_ava("uuid", &Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d2")); e2.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d2"));
let e2 = unsafe { e2.into_sealed_new() }; let e2 = unsafe { e2.into_sealed_new() };
let _rset = be.create(audit, vec![e1.clone(), e2.clone()]).unwrap(); let _rset = be.create(audit, vec![e1.clone(), e2.clone()]).unwrap();

2
kanidmd/src/lib/constants/mod.rs
View file

@ -13,7 +13,7 @@ pub use crate::constants::system_config::*;
pub use crate::constants::uuids::*; pub use crate::constants::uuids::*;
// Increment this as we add new schema types and values!!! // Increment this as we add new schema types and values!!!
pub const SYSTEM_INDEX_VERSION: i64 = 10; pub const SYSTEM_INDEX_VERSION: i64 = 11;
// On test builds, define to 60 seconds // On test builds, define to 60 seconds
#[cfg(test)] #[cfg(test)]
pub const PURGE_FREQUENCY: u64 = 60; pub const PURGE_FREQUENCY: u64 = 60;

2
kanidmd/src/lib/constants/schema.rs
View file

@ -320,7 +320,7 @@ pub const JSON_SCHEMA_ATTR_BADLIST_PASSWORD: &str = r#"{
"EQUALITY" "EQUALITY"
], ],
"unique": [ "unique": [
"true" "false"
], ],
"multivalue": [ "multivalue": [
"true" "true"

2
kanidmd/src/lib/core/mod.rs
View file

@ -427,7 +427,7 @@ async fn json_rest_event_credential_put(
// Okay, so a put normally needs // Okay, so a put normally needs
// * filter of what we are working on (id + class) // * filter of what we are working on (id + class)
// * a BTreeMap<String, Vec<String>> that we turn into a modlist. // * a Map<String, Vec<String>> that we turn into a modlist.
// //
// OR // OR
// * filter of what we are working on (id + class) // * filter of what we are working on (id + class)

135
kanidmd/src/lib/entry.rs
View file

@ -48,10 +48,9 @@ use crate::be::IdxKey;
use ldap3_server::simple::{LdapPartialAttribute, LdapSearchResultEntry}; use ldap3_server::simple::{LdapPartialAttribute, LdapSearchResultEntry};
use std::collections::BTreeSet as Set; use std::collections::BTreeSet as Set;
use std::collections::BTreeSet; use std::collections::BTreeSet;
// BTreeMap could be faster, but it's small datasets? // use std::collections::BTreeMap as Map;
// use std::collections::HashMap as Map; use hashbrown::HashMap as Map;
use std::collections::BTreeMap as Map; use hashbrown::HashSet;
use std::collections::HashSet;
use uuid::Uuid; use uuid::Uuid;
// use std::convert::TryFrom; // use std::convert::TryFrom;
@ -246,7 +245,7 @@ impl Entry<EntryInit, EntryNew> {
// This means NEVER COMMITED // This means NEVER COMMITED
valid: EntryInit, valid: EntryInit,
state: EntryNew, state: EntryNew,
attrs: Map::new(), attrs: Map::with_capacity(32),
} }
} }
@ -486,7 +485,7 @@ impl Entry<EntryInit, EntryNew> {
} }
#[cfg(test)] #[cfg(test)]
pub fn add_ava(&mut self, attr: &str, value: &Value) { pub fn add_ava(&mut self, attr: &str, value: Value) {
self.add_ava_int(attr, value) self.add_ava_int(attr, value)
} }
} }
@ -721,7 +720,7 @@ impl Entry<EntryInvalid, EntryCommitted> {
} }
pub fn into_recycled(mut self) -> Self { pub fn into_recycled(mut self) -> Self {
self.add_ava("class", &Value::new_class("recycled")); self.add_ava("class", Value::new_class("recycled"));
Entry { Entry {
valid: self.valid.clone(), valid: self.valid.clone(),
@ -1475,19 +1474,12 @@ impl Entry<EntryReduced, EntryCommitted> {
// impl<STATE> Entry<EntryValid, STATE> { // impl<STATE> Entry<EntryValid, STATE> {
impl<VALID, STATE> Entry<VALID, STATE> { impl<VALID, STATE> Entry<VALID, STATE> {
fn add_ava_int(&mut self, attr: &str, value: &Value) { fn add_ava_int(&mut self, attr: &str, value: Value) {
// How do we make this turn into an ok / err? // How do we make this turn into an ok / err?
self.attrs let v = self.attrs.entry(attr.to_string()).or_insert_with(Set::new);
.entry(attr.to_string())
.and_modify(|v| {
// Here we need to actually do a check/binary search ... // Here we need to actually do a check/binary search ...
if v.contains(value) { v.insert(value);
// It already exists, done! // Doesn't matter if it already exists, equality will replace.
} else {
v.insert(value.clone());
}
})
.or_insert(btreeset![value.clone()]);
} }
fn set_last_changed(&mut self, cid: Cid) { fn set_last_changed(&mut self, cid: Cid) {
@ -1699,6 +1691,12 @@ impl<VALID, STATE> Entry<VALID, STATE> {
acc acc
} }
}), }),
FilterResolved::Inclusion(_) => {
// An inclusion doesn't make sense on an entry in isolation!
// Inclusions are part of exists queries, on search they mean
// nothing!
false
}
FilterResolved::AndNot(f) => !self.entry_match_no_index_inner(f), FilterResolved::AndNot(f) => !self.entry_match_no_index_inner(f),
} }
} }
@ -1796,7 +1794,10 @@ where
// a list of syntax violations ... // a list of syntax violations ...
// If this already exists, we silently drop the event? Is that an // If this already exists, we silently drop the event? Is that an
// acceptable interface? // acceptable interface?
pub fn add_ava(&mut self, attr: &str, value: &Value) { //
// TODO: This should take Value not &Value, would save a lot of clones
// around the codebase.
pub fn add_ava(&mut self, attr: &str, value: Value) {
self.add_ava_int(attr, value) self.add_ava_int(attr, value)
} }
@ -1809,6 +1810,15 @@ where
}); });
} }
// Need something that can remove by difference?
pub(crate) fn remove_avas(&mut self, attr: &str, values: &BTreeSet<PartialValue>) {
if let Some(set) = self.attrs.get_mut(attr) {
values.iter().for_each(|k| {
set.remove(k);
})
}
}
pub fn purge_ava(&mut self, attr: &str) { pub fn purge_ava(&mut self, attr: &str) {
self.attrs.remove(attr); self.attrs.remove(attr);
} }
@ -1817,13 +1827,6 @@ where
self.attrs.remove(attr) self.attrs.remove(attr)
} }
/// Overwrite the existing avas.
pub fn set_avas(&mut self, attr: &str, values: Vec<Value>) {
// Overwrite the existing value, build a tree from the list.
let x: Set<_> = values.into_iter().collect();
let _ = self.attrs.insert(attr.to_string(), x);
}
/// Provide a true ava set. /// Provide a true ava set.
pub fn set_ava(&mut self, attr: &str, values: Set<Value>) { pub fn set_ava(&mut self, attr: &str, values: Set<Value>) {
// Overwrite the existing value, build a tree from the list. // Overwrite the existing value, build a tree from the list.
@ -1848,7 +1851,7 @@ where
// mutate // mutate
for modify in modlist { for modify in modlist {
match modify { match modify {
Modify::Present(a, v) => self.add_ava(a.as_str(), v), Modify::Present(a, v) => self.add_ava(a.as_str(), v.clone()),
Modify::Removed(a, v) => self.remove_ava(a.as_str(), v), Modify::Removed(a, v) => self.remove_ava(a.as_str(), v),
Modify::Purged(a) => self.purge_ava(a.as_str()), Modify::Purged(a) => self.purge_ava(a.as_str()),
} }
@ -1888,7 +1891,7 @@ impl From<&SchemaAttribute> for Entry<EntryInit, EntryNew> {
let syntax_v = btreeset![Value::from(s.syntax.clone())]; let syntax_v = btreeset![Value::from(s.syntax.clone())];
// Build the Map of the attributes relevant // Build the Map of the attributes relevant
let mut attrs: Map<String, Set<Value>> = Map::new(); let mut attrs: Map<String, Set<Value>> = Map::with_capacity(16);
attrs.insert("attributename".to_string(), name_v); attrs.insert("attributename".to_string(), name_v);
attrs.insert("description".to_string(), desc_v); attrs.insert("description".to_string(), desc_v);
attrs.insert("uuid".to_string(), uuid_v); attrs.insert("uuid".to_string(), uuid_v);
@ -1922,7 +1925,7 @@ impl From<&SchemaClass> for Entry<EntryInit, EntryNew> {
let name_v = btreeset![Value::new_iutf8(s.name.clone())]; let name_v = btreeset![Value::new_iutf8(s.name.clone())];
let desc_v = btreeset![Value::new_utf8(s.description.clone())]; let desc_v = btreeset![Value::new_utf8(s.description.clone())];
let mut attrs: Map<String, Set<Value>> = Map::new(); let mut attrs: Map<String, Set<Value>> = Map::with_capacity(16);
attrs.insert("classname".to_string(), name_v); attrs.insert("classname".to_string(), name_v);
attrs.insert("description".to_string(), desc_v); attrs.insert("description".to_string(), desc_v);
attrs.insert("uuid".to_string(), uuid_v); attrs.insert("uuid".to_string(), uuid_v);
@ -1969,14 +1972,14 @@ mod tests {
use crate::entry::{Entry, EntryInit, EntryInvalid, EntryNew}; use crate::entry::{Entry, EntryInit, EntryInvalid, EntryNew};
use crate::modify::{Modify, ModifyList}; use crate::modify::{Modify, ModifyList};
use crate::value::{IndexType, PartialValue, Value}; use crate::value::{IndexType, PartialValue, Value};
use hashbrown::HashSet;
use std::collections::BTreeSet as Set; use std::collections::BTreeSet as Set;
use std::collections::HashSet;
#[test] #[test]
fn test_entry_basic() { fn test_entry_basic() {
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
} }
#[test] #[test]
@ -1988,8 +1991,8 @@ mod tests {
// are adding ... Or do we validate after the changes are made in // are adding ... Or do we validate after the changes are made in
// total? // total?
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
let values = e.get_ava_set("userid").expect("Failed to get ava"); let values = e.get_ava_set("userid").expect("Failed to get ava");
// Should only be one value! // Should only be one value!
@ -1999,7 +2002,7 @@ mod tests {
#[test] #[test]
fn test_entry_pres() { fn test_entry_pres() {
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
assert!(e.attribute_pres("userid")); assert!(e.attribute_pres("userid"));
assert!(!e.attribute_pres("name")); assert!(!e.attribute_pres("name"));
@ -2009,7 +2012,7 @@ mod tests {
fn test_entry_equality() { fn test_entry_equality() {
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
assert!(e.attribute_equality("userid", &PartialValue::new_utf8s("william"))); assert!(e.attribute_equality("userid", &PartialValue::new_utf8s("william")));
assert!(!e.attribute_equality("userid", &PartialValue::new_utf8s("test"))); assert!(!e.attribute_equality("userid", &PartialValue::new_utf8s("test")));
@ -2022,7 +2025,7 @@ mod tests {
fn test_entry_substring() { fn test_entry_substring() {
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
assert!(e.attribute_substring("userid", &PartialValue::new_utf8s("william"))); assert!(e.attribute_substring("userid", &PartialValue::new_utf8s("william")));
assert!(e.attribute_substring("userid", &PartialValue::new_utf8s("will"))); assert!(e.attribute_substring("userid", &PartialValue::new_utf8s("will")));
@ -2042,14 +2045,14 @@ mod tests {
let pv10 = PartialValue::new_uint32(10); let pv10 = PartialValue::new_uint32(10);
let pv15 = PartialValue::new_uint32(15); let pv15 = PartialValue::new_uint32(15);
e1.add_ava("a", &Value::new_uint32(10)); e1.add_ava("a", Value::new_uint32(10));
assert!(e1.attribute_lessthan("a", &pv2) == false); assert!(e1.attribute_lessthan("a", &pv2) == false);
assert!(e1.attribute_lessthan("a", &pv8) == false); assert!(e1.attribute_lessthan("a", &pv8) == false);
assert!(e1.attribute_lessthan("a", &pv10) == false); assert!(e1.attribute_lessthan("a", &pv10) == false);
assert!(e1.attribute_lessthan("a", &pv15) == true); assert!(e1.attribute_lessthan("a", &pv15) == true);
e1.add_ava("a", &Value::new_uint32(8)); e1.add_ava("a", Value::new_uint32(8));
assert!(e1.attribute_lessthan("a", &pv2) == false); assert!(e1.attribute_lessthan("a", &pv2) == false);
assert!(e1.attribute_lessthan("a", &pv8) == false); assert!(e1.attribute_lessthan("a", &pv8) == false);
@ -2062,7 +2065,7 @@ mod tests {
// Test application of changes to an entry. // Test application of changes to an entry.
let mut e: Entry<EntryInvalid, EntryNew> = unsafe { Entry::new().into_invalid_new() }; let mut e: Entry<EntryInvalid, EntryNew> = unsafe { Entry::new().into_invalid_new() };
e.add_ava("userid", &Value::from("william")); e.add_ava("userid", Value::from("william"));
let present_single_mods = unsafe { let present_single_mods = unsafe {
ModifyList::new_valid_list(vec![Modify::Present( ModifyList::new_valid_list(vec![Modify::Present(
@ -2132,18 +2135,18 @@ mod tests {
#[test] #[test]
fn test_entry_idx_diff() { fn test_entry_idx_diff() {
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("userid", &Value::from("william")); e1.add_ava("userid", Value::from("william"));
let mut e1_mod = e1.clone(); let mut e1_mod = e1.clone();
e1_mod.add_ava("extra", &Value::from("test")); e1_mod.add_ava("extra", Value::from("test"));
let e1 = unsafe { e1.into_sealed_committed() }; let e1 = unsafe { e1.into_sealed_committed() };
let e1_mod = unsafe { e1_mod.into_sealed_committed() }; let e1_mod = unsafe { e1_mod.into_sealed_committed() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("userid", &Value::from("claire")); e2.add_ava("userid", Value::from("claire"));
let e2 = unsafe { e2.into_sealed_committed() }; let e2 = unsafe { e2.into_sealed_committed() };
let mut idxmeta = HashSet::new(); let mut idxmeta = HashSet::with_capacity(8);
idxmeta.insert(IdxKey { idxmeta.insert(IdxKey {
attr: "userid".to_string(), attr: "userid".to_string(),
itype: IndexType::EQUALITY, itype: IndexType::EQUALITY,
@ -2244,18 +2247,18 @@ mod tests {
#[test] #[test]
fn test_entry_mask_recycled_ts() { fn test_entry_mask_recycled_ts() {
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("class", &Value::new_class("person")); e1.add_ava("class", Value::new_class("person"));
let e1 = unsafe { e1.into_sealed_committed() }; let e1 = unsafe { e1.into_sealed_committed() };
assert!(e1.mask_recycled_ts().is_some()); assert!(e1.mask_recycled_ts().is_some());
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("class", &Value::new_class("person")); e2.add_ava("class", Value::new_class("person"));
e2.add_ava("class", &Value::new_class("recycled")); e2.add_ava("class", Value::new_class("recycled"));
let e2 = unsafe { e2.into_sealed_committed() }; let e2 = unsafe { e2.into_sealed_committed() };
assert!(e2.mask_recycled_ts().is_none()); assert!(e2.mask_recycled_ts().is_none());
let mut e3: Entry<EntryInit, EntryNew> = Entry::new(); let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
e3.add_ava("class", &Value::new_class("tombstone")); e3.add_ava("class", Value::new_class("tombstone"));
let e3 = unsafe { e3.into_sealed_committed() }; let e3 = unsafe { e3.into_sealed_committed() };
assert!(e3.mask_recycled_ts().is_none()); assert!(e3.mask_recycled_ts().is_none());
} }
@ -2269,7 +2272,7 @@ mod tests {
// none, some - test adding an entry gives back add sets // none, some - test adding an entry gives back add sets
{ {
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("class", &Value::new_class("person")); e.add_ava("class", Value::new_class("person"));
let e = unsafe { e.into_sealed_committed() }; let e = unsafe { e.into_sealed_committed() };
assert!(Entry::idx_name2uuid_diff(None, Some(&e)) == (Some(Set::new()), None)); assert!(Entry::idx_name2uuid_diff(None, Some(&e)) == (Some(Set::new()), None));
@ -2277,13 +2280,13 @@ mod tests {
{ {
let mut e: Entry<EntryInit, EntryNew> = Entry::new(); let mut e: Entry<EntryInit, EntryNew> = Entry::new();
e.add_ava("class", &Value::new_class("person")); e.add_ava("class", Value::new_class("person"));
e.add_ava("gidnumber", &Value::new_uint32(1300)); e.add_ava("gidnumber", Value::new_uint32(1300));
e.add_ava("name", &Value::new_iname_s("testperson")); e.add_ava("name", Value::new_iname_s("testperson"));
e.add_ava("spn", &Value::new_spn_str("testperson", "example.com")); e.add_ava("spn", Value::new_spn_str("testperson", "example.com"));
e.add_ava( e.add_ava(
"uuid", "uuid",
&Value::new_uuids("9fec0398-c46c-4df4-9df5-b0016f7d563f").unwrap(), Value::new_uuids("9fec0398-c46c-4df4-9df5-b0016f7d563f").unwrap(),
); );
let e = unsafe { e.into_sealed_committed() }; let e = unsafe { e.into_sealed_committed() };
@ -2323,14 +2326,14 @@ mod tests {
{ {
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("class", &Value::new_class("person")); e1.add_ava("class", Value::new_class("person"));
e1.add_ava("spn", &Value::new_spn_str("testperson", "example.com")); e1.add_ava("spn", Value::new_spn_str("testperson", "example.com"));
let e1 = unsafe { e1.into_sealed_committed() }; let e1 = unsafe { e1.into_sealed_committed() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("class", &Value::new_class("person")); e2.add_ava("class", Value::new_class("person"));
e2.add_ava("name", &Value::new_iname_s("testperson")); e2.add_ava("name", Value::new_iname_s("testperson"));
e2.add_ava("spn", &Value::new_spn_str("testperson", "example.com")); e2.add_ava("spn", Value::new_spn_str("testperson", "example.com"));
let e2 = unsafe { e2.into_sealed_committed() }; let e2 = unsafe { e2.into_sealed_committed() };
// One attr added // One attr added
@ -2349,13 +2352,13 @@ mod tests {
// Value changed, remove old, add new. // Value changed, remove old, add new.
{ {
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("class", &Value::new_class("person")); e1.add_ava("class", Value::new_class("person"));
e1.add_ava("spn", &Value::new_spn_str("testperson", "example.com")); e1.add_ava("spn", Value::new_spn_str("testperson", "example.com"));
let e1 = unsafe { e1.into_sealed_committed() }; let e1 = unsafe { e1.into_sealed_committed() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("class", &Value::new_class("person")); e2.add_ava("class", Value::new_class("person"));
e2.add_ava("spn", &Value::new_spn_str("renameperson", "example.com")); e2.add_ava("spn", Value::new_spn_str("renameperson", "example.com"));
let e2 = unsafe { e2.into_sealed_committed() }; let e2 = unsafe { e2.into_sealed_committed() };
assert!( assert!(
@ -2373,11 +2376,11 @@ mod tests {
assert!(Entry::idx_uuid2spn_diff(None, None) == None); assert!(Entry::idx_uuid2spn_diff(None, None) == None);
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("spn", &Value::new_spn_str("testperson", "example.com")); e1.add_ava("spn", Value::new_spn_str("testperson", "example.com"));
let e1 = unsafe { e1.into_sealed_committed() }; let e1 = unsafe { e1.into_sealed_committed() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("spn", &Value::new_spn_str("renameperson", "example.com")); e2.add_ava("spn", Value::new_spn_str("renameperson", "example.com"));
let e2 = unsafe { e2.into_sealed_committed() }; let e2 = unsafe { e2.into_sealed_committed() };
assert!( assert!(
@ -2397,11 +2400,11 @@ mod tests {
assert!(Entry::idx_uuid2rdn_diff(None, None) == None); assert!(Entry::idx_uuid2rdn_diff(None, None) == None);
let mut e1: Entry<EntryInit, EntryNew> = Entry::new(); let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
e1.add_ava("spn", &Value::new_spn_str("testperson", "example.com")); e1.add_ava("spn", Value::new_spn_str("testperson", "example.com"));
let e1 = unsafe { e1.into_sealed_committed() }; let e1 = unsafe { e1.into_sealed_committed() };
let mut e2: Entry<EntryInit, EntryNew> = Entry::new(); let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
e2.add_ava("spn", &Value::new_spn_str("renameperson", "example.com")); e2.add_ava("spn", Value::new_spn_str("renameperson", "example.com"));
let e2 = unsafe { e2.into_sealed_committed() }; let e2 = unsafe { e2.into_sealed_committed() };
assert!( assert!(

67
kanidmd/src/lib/filter.rs
View file

@ -17,12 +17,12 @@ use crate::server::{
QueryServerReadTransaction, QueryServerTransaction, QueryServerWriteTransaction, QueryServerReadTransaction, QueryServerTransaction, QueryServerWriteTransaction,
}; };
use crate::value::{IndexType, PartialValue}; use crate::value::{IndexType, PartialValue};
use hashbrown::HashSet;
use kanidm_proto::v1::Filter as ProtoFilter; use kanidm_proto::v1::Filter as ProtoFilter;
use kanidm_proto::v1::{OperationError, SchemaError}; use kanidm_proto::v1::{OperationError, SchemaError};
use ldap3_server::simple::LdapFilter; use ldap3_server::simple::LdapFilter;
use std::cmp::{Ordering, PartialOrd}; use std::cmp::{Ordering, PartialOrd};
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::collections::HashSet;
use std::iter; use std::iter;
use uuid::Uuid; use uuid::Uuid;
@ -61,6 +61,11 @@ pub fn f_and(vs: Vec<FC>) -> FC {
FC::And(vs) FC::And(vs)
} }
#[allow(dead_code)]
pub fn f_inc(vs: Vec<FC>) -> FC {
FC::Inclusion(vs)
}
#[allow(dead_code)] #[allow(dead_code)]
pub fn f_andnot(fc: FC) -> FC { pub fn f_andnot(fc: FC) -> FC {
FC::AndNot(Box::new(fc)) FC::AndNot(Box::new(fc))
@ -107,6 +112,7 @@ pub enum FC<'a> {
LessThan(&'a str, PartialValue), LessThan(&'a str, PartialValue),
Or(Vec<FC<'a>>), Or(Vec<FC<'a>>),
And(Vec<FC<'a>>), And(Vec<FC<'a>>),
Inclusion(Vec<FC<'a>>),
AndNot(Box<FC<'a>>), AndNot(Box<FC<'a>>),
SelfUUID, SelfUUID,
// Not(Box<FC>), // Not(Box<FC>),
@ -122,6 +128,7 @@ enum FilterComp {
LessThan(String, PartialValue), LessThan(String, PartialValue),
Or(Vec<FilterComp>), Or(Vec<FilterComp>),
And(Vec<FilterComp>), And(Vec<FilterComp>),
Inclusion(Vec<FilterComp>),
AndNot(Box<FilterComp>), AndNot(Box<FilterComp>),
SelfUUID, SelfUUID,
// Does this mean we can add a true not to the type now? // Does this mean we can add a true not to the type now?
@ -141,6 +148,8 @@ pub enum FilterResolved {
LessThan(String, PartialValue, bool), LessThan(String, PartialValue, bool),
Or(Vec<FilterResolved>), Or(Vec<FilterResolved>),
And(Vec<FilterResolved>), And(Vec<FilterResolved>),
// All terms must have 1 or more items, or the inclusion is false!
Inclusion(Vec<FilterResolved>),
AndNot(Box<FilterResolved>), AndNot(Box<FilterResolved>),
} }
@ -182,6 +191,8 @@ pub enum FilterPlan {
AndPartial(Vec<FilterPlan>), AndPartial(Vec<FilterPlan>),
AndPartialThreshold(Vec<FilterPlan>), AndPartialThreshold(Vec<FilterPlan>),
AndNot(Box<FilterPlan>), AndNot(Box<FilterPlan>),
InclusionInvalid(Vec<FilterPlan>),
InclusionIndexed(Vec<FilterPlan>),
} }
/// A `Filter` is a logical set of assertions about the state of an [`Entry`] and /// A `Filter` is a logical set of assertions about the state of an [`Entry`] and
@ -285,7 +296,7 @@ impl Filter<FilterValid> {
pub fn get_attr_set(&self) -> BTreeSet<&str> { pub fn get_attr_set(&self) -> BTreeSet<&str> {
// Recurse through the filter getting an attribute set. // Recurse through the filter getting an attribute set.
let mut r_set: BTreeSet<&str> = BTreeSet::new(); let mut r_set = BTreeSet::new();
self.state.inner.get_attr_set(&mut r_set); self.state.inner.get_attr_set(&mut r_set);
r_set r_set
} }
@ -473,6 +484,7 @@ impl FilterComp {
FC::LessThan(a, v) => FilterComp::LessThan(a.to_string(), v), FC::LessThan(a, v) => FilterComp::LessThan(a.to_string(), v),
FC::Or(v) => FilterComp::Or(v.into_iter().map(FilterComp::new).collect()), FC::Or(v) => FilterComp::Or(v.into_iter().map(FilterComp::new).collect()),
FC::And(v) => FilterComp::And(v.into_iter().map(FilterComp::new).collect()), FC::And(v) => FilterComp::And(v.into_iter().map(FilterComp::new).collect()),
FC::Inclusion(v) => FilterComp::Inclusion(v.into_iter().map(FilterComp::new).collect()),
FC::AndNot(b) => FilterComp::AndNot(Box::new(FilterComp::new(*b))), FC::AndNot(b) => FilterComp::AndNot(Box::new(FilterComp::new(*b))),
FC::SelfUUID => FilterComp::SelfUUID, FC::SelfUUID => FilterComp::SelfUUID,
} }
@ -511,6 +523,7 @@ impl FilterComp {
} }
FilterComp::Or(vs) => vs.iter().for_each(|f| f.get_attr_set(r_set)), FilterComp::Or(vs) => vs.iter().for_each(|f| f.get_attr_set(r_set)),
FilterComp::And(vs) => vs.iter().for_each(|f| f.get_attr_set(r_set)), FilterComp::And(vs) => vs.iter().for_each(|f| f.get_attr_set(r_set)),
FilterComp::Inclusion(vs) => vs.iter().for_each(|f| f.get_attr_set(r_set)),
FilterComp::AndNot(f) => f.get_attr_set(r_set), FilterComp::AndNot(f) => f.get_attr_set(r_set),
FilterComp::SelfUUID => { FilterComp::SelfUUID => {
r_set.insert("uuid"); r_set.insert("uuid");
@ -615,6 +628,17 @@ impl FilterComp {
// Now put the valid filters into the Filter // Now put the valid filters into the Filter
x.map(FilterComp::And) x.map(FilterComp::And)
} }
FilterComp::Inclusion(filters) => {
if filters.is_empty() {
return Err(SchemaError::EmptyFilter);
};
let x: Result<Vec<_>, _> = filters
.iter()
.map(|filter| filter.validate(schema))
.collect();
// Now put the valid filters into the Filter
x.map(FilterComp::Inclusion)
}
FilterComp::AndNot(filter) => { FilterComp::AndNot(filter) => {
// Just validate the inner // Just validate the inner
filter filter
@ -859,6 +883,11 @@ impl FilterResolved {
.map(|v| FilterResolved::from_invalid(v, idxmeta)) .map(|v| FilterResolved::from_invalid(v, idxmeta))
.collect(), .collect(),
), ),
FilterComp::Inclusion(vs) => FilterResolved::Inclusion(
vs.into_iter()
.map(|v| FilterResolved::from_invalid(v, idxmeta))
.collect(),
),
FilterComp::AndNot(f) => { FilterComp::AndNot(f) => {
// TODO: pattern match box here. (AndNot(box f)). // TODO: pattern match box here. (AndNot(box f)).
// We have to clone f into our space here because pattern matching can // We have to clone f into our space here because pattern matching can
@ -911,6 +940,13 @@ impl FilterResolved {
.collect(); .collect();
fi.map(FilterResolved::And) fi.map(FilterResolved::And)
} }
FilterComp::Inclusion(vs) => {
let fi: Option<Vec<_>> = vs
.into_iter()
.map(|f| FilterResolved::resolve_idx(f, ev, idxmeta))
.collect();
fi.map(FilterResolved::Inclusion)
}
FilterComp::AndNot(f) => { FilterComp::AndNot(f) => {
// TODO: pattern match box here. (AndNot(box f)). // TODO: pattern match box here. (AndNot(box f)).
// We have to clone f into our space here because pattern matching can // We have to clone f into our space here because pattern matching can
@ -955,6 +991,13 @@ impl FilterResolved {
.collect(); .collect();
fi.map(FilterResolved::And) fi.map(FilterResolved::And)
} }
FilterComp::Inclusion(vs) => {
let fi: Option<Vec<_>> = vs
.into_iter()
.map(|f| FilterResolved::resolve_no_idx(f, ev))
.collect();
fi.map(FilterResolved::Inclusion)
}
FilterComp::AndNot(f) => { FilterComp::AndNot(f) => {
// TODO: pattern match box here. (AndNot(box f)). // TODO: pattern match box here. (AndNot(box f)).
// We have to clone f into our space here because pattern matching can // We have to clone f into our space here because pattern matching can
@ -978,6 +1021,26 @@ impl FilterResolved {
fn optimise(&self) -> Self { fn optimise(&self) -> Self {
// Most optimisations only matter around or/and terms. // Most optimisations only matter around or/and terms.
match self { match self {
FilterResolved::Inclusion(f_list) => {
// first, optimise all our inner elements
let (f_list_inc, mut f_list_new): (Vec<_>, Vec<_>) = f_list
.iter()
.map(|f_ref| f_ref.optimise())
.partition(|f| match f {
FilterResolved::Inclusion(_) => true,
_ => false,
});
f_list_inc.into_iter().for_each(|fc| {
if let FilterResolved::Inclusion(mut l) = fc {
f_list_new.append(&mut l)
}
});
// finally, optimise this list by sorting.
f_list_new.sort_unstable();
f_list_new.dedup();
FilterResolved::Inclusion(f_list_new)
}
FilterResolved::And(f_list) => { FilterResolved::And(f_list) => {
// first, optimise all our inner elements // first, optimise all our inner elements
let (f_list_and, mut f_list_new): (Vec<_>, Vec<_>) = f_list let (f_list_and, mut f_list_new): (Vec<_>, Vec<_>) = f_list

28
kanidmd/src/lib/macros.rs
View file

@ -104,7 +104,7 @@ macro_rules! entry_str_to_account {
.get_ava_single_str("name") .get_ava_single_str("name")
.map(|s| Value::new_spn_str(s, "example.com")) .map(|s| Value::new_spn_str(s, "example.com"))
.expect("Failed to munge spn from name!"); .expect("Failed to munge spn from name!");
e.set_avas("spn", vec![spn]); e.set_ava("spn", btreeset![spn]);
let e = unsafe { e.into_sealed_committed() }; let e = unsafe { e.into_sealed_committed() };
@ -180,6 +180,18 @@ macro_rules! f_and {
}}; }};
} }
#[allow(unused_macros)]
#[macro_export]
macro_rules! f_inc {
(
$vs:expr
) => {{
use crate::filter::FC;
let s: Box<[FC]> = Box::new($vs);
f_inc(s.into_vec())
}};
}
#[allow(unused_macros)] #[allow(unused_macros)]
#[macro_export] #[macro_export]
macro_rules! f_or { macro_rules! f_or {
@ -203,7 +215,7 @@ macro_rules! filter {
use crate::filter::FC; use crate::filter::FC;
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::filter::{ use crate::filter::{
f_and, f_andnot, f_eq, f_id, f_lt, f_or, f_pres, f_self, f_spn_name, f_sub, f_and, f_andnot, f_eq, f_id, f_inc, f_lt, f_or, f_pres, f_self, f_spn_name, f_sub,
}; };
Filter::new_ignore_hidden($fc) Filter::new_ignore_hidden($fc)
}}; }};
@ -219,7 +231,9 @@ macro_rules! filter_rec {
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::filter::FC; use crate::filter::FC;
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::filter::{f_and, f_andnot, f_eq, f_id, f_lt, f_or, f_pres, f_self, f_sub}; use crate::filter::{
f_and, f_andnot, f_eq, f_id, f_inc, f_lt, f_or, f_pres, f_self, f_sub,
};
Filter::new_recycled($fc) Filter::new_recycled($fc)
}}; }};
} }
@ -234,7 +248,9 @@ macro_rules! filter_all {
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::filter::FC; use crate::filter::FC;
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::filter::{f_and, f_andnot, f_eq, f_id, f_lt, f_or, f_pres, f_self, f_sub}; use crate::filter::{
f_and, f_andnot, f_eq, f_id, f_inc, f_lt, f_or, f_pres, f_self, f_sub,
};
Filter::new($fc) Filter::new($fc)
}}; }};
} }
@ -247,7 +263,7 @@ macro_rules! filter_valid {
$fc:expr $fc:expr
) => {{ ) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::filter::{f_and, f_andnot, f_eq, f_lt, f_or, f_pres, f_sub}; use crate::filter::{f_and, f_andnot, f_eq, f_inc, f_lt, f_or, f_pres, f_sub};
use crate::filter::{Filter, FilterInvalid}; use crate::filter::{Filter, FilterInvalid};
let f: Filter<FilterInvalid> = Filter::new($fc); let f: Filter<FilterInvalid> = Filter::new($fc);
// Create a resolved filter, via the most unsafe means possible! // Create a resolved filter, via the most unsafe means possible!
@ -263,7 +279,7 @@ macro_rules! filter_resolved {
$fc:expr $fc:expr
) => {{ ) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::filter::{f_and, f_andnot, f_eq, f_lt, f_or, f_pres, f_sub}; use crate::filter::{f_and, f_andnot, f_eq, f_inc, f_lt, f_or, f_pres, f_sub};
use crate::filter::{Filter, FilterInvalid}; use crate::filter::{Filter, FilterInvalid};
let f: Filter<FilterInvalid> = Filter::new($fc); let f: Filter<FilterInvalid> = Filter::new($fc);
// Create a resolved filter, via the most unsafe means possible! // Create a resolved filter, via the most unsafe means possible!

35
kanidmd/src/lib/plugins/attrunique.rs
View file

@ -26,7 +26,8 @@ fn get_cand_attr_set<VALID, STATE>(
) -> Result<BTreeMap<PartialValue, PartialValue>, OperationError> { ) -> Result<BTreeMap<PartialValue, PartialValue>, OperationError> {
let mut cand_attr: BTreeMap<PartialValue, PartialValue> = BTreeMap::new(); let mut cand_attr: BTreeMap<PartialValue, PartialValue> = BTreeMap::new();
for e in cand.iter() { cand.iter()
.try_for_each(|e| {
let uuid = match e.get_ava_single("uuid") { let uuid = match e.get_ava_single("uuid") {
Some(v) => v.to_partialvalue(), Some(v) => v.to_partialvalue(),
None => { None => {
@ -35,22 +36,11 @@ fn get_cand_attr_set<VALID, STATE>(
}; };
// Get the value and uuid // Get the value and uuid
//for each value in the ava. //for each value in the ava.
let mut values: Vec<PartialValue> = match e.get_ava(attr) { e.get_ava(attr)
Some(vs) => { .map(|vs| {
// We have values, map them. vs.map(|v| v.to_partialvalue()).try_for_each(|v| {
vs.map(|v| v.to_partialvalue()).collect()
}
None => {
// No values, so empty set.
Vec::new()
}
};
for v in values.drain(..) {
match cand_attr.insert(v, uuid.clone()) { match cand_attr.insert(v, uuid.clone()) {
// Didn't exist, move on. None => Ok(()),
None => {}
// The duplicate/rejected value moved out of the tree
Some(vr) => { Some(vr) => {
ladmin_error!( ladmin_error!(
au, au,
@ -59,15 +49,16 @@ fn get_cand_attr_set<VALID, STATE>(
vr, vr,
uuid uuid
); );
return Err(OperationError::Plugin(PluginError::AttrUnique( Err(OperationError::Plugin(PluginError::AttrUnique(
"ava already exists".to_string(), "ava already exists".to_string(),
))); )))
} }
} }
} })
} })
.unwrap_or(Ok(()))
Ok(cand_attr) })
.map(|()| cand_attr)
} }
fn enforce_unique<STATE>( fn enforce_unique<STATE>(

27
kanidmd/src/lib/plugins/base.rs
View file

@ -1,4 +1,5 @@
use crate::plugins::Plugin; use crate::plugins::Plugin;
use hashbrown::HashSet;
use std::collections::BTreeSet; use std::collections::BTreeSet;
use uuid::Uuid; use uuid::Uuid;
@ -56,7 +57,7 @@ impl Plugin for Base {
ltrace!(au, "Base check on entry: {:?}", entry); ltrace!(au, "Base check on entry: {:?}", entry);
// First, ensure we have the 'object', class in the class set. // First, ensure we have the 'object', class in the class set.
entry.add_ava("class", &CLASS_OBJECT); entry.add_ava("class", CLASS_OBJECT.clone());
ltrace!(au, "Object should now be in entry: {:?}", entry); ltrace!(au, "Object should now be in entry: {:?}", entry);
@ -66,9 +67,9 @@ impl Plugin for Base {
match entry.get_ava_set("uuid").map(|s| s.len()) { match entry.get_ava_set("uuid").map(|s| s.len()) {
None => { None => {
// Generate // Generate
let ava_uuid: Vec<Value> = vec![Value::new_uuid(Uuid::new_v4())]; let ava_uuid = btreeset![Value::new_uuid(Uuid::new_v4())];
ltrace!(au, "Setting temporary UUID {:?} to entry", ava_uuid[0]); ltrace!(au, "Setting temporary UUID {:?} to entry", ava_uuid);
entry.set_avas("uuid", ava_uuid); entry.set_ava("uuid", ava_uuid);
} }
Some(1) => { Some(1) => {
// Do nothing // Do nothing
@ -208,9 +209,6 @@ impl Plugin for Base {
au: &mut AuditScope, au: &mut AuditScope,
qs: &QueryServerReadTransaction, qs: &QueryServerReadTransaction,
) -> Vec<Result<(), ConsistencyError>> { ) -> Vec<Result<(), ConsistencyError>> {
// Verify all uuid's are unique?
// Probably the literally worst thing ...
// Search for class = * // Search for class = *
let entries = match qs.internal_search(au, filter!(f_pres("class"))) { let entries = match qs.internal_search(au, filter!(f_pres("class"))) {
Ok(v) => v, Ok(v) => v,
@ -220,6 +218,8 @@ impl Plugin for Base {
} }
}; };
let mut uuid_seen: HashSet<Uuid> = HashSet::with_capacity(entries.len());
entries entries
.iter() .iter()
// do an exists checks on the uuid // do an exists checks on the uuid
@ -228,21 +228,14 @@ impl Plugin for Base {
// will be thrown in the deserialise (possibly it will be better // will be thrown in the deserialise (possibly it will be better
// handled later). But it means this check only needs to validate // handled later). But it means this check only needs to validate
// uniqueness! // uniqueness!
let uuid: &Uuid = e.get_uuid(); let uuid = e.get_uuid();
let filt = filter!(FC::Eq("uuid", PartialValue::new_uuid(*uuid))); if uuid_seen.insert(*uuid) {
match qs.internal_search(au, filt) { // Insert returns true if the item was unique.
Ok(r) => {
if r.is_empty() {
Err(ConsistencyError::UuidIndexCorrupt(uuid.to_string()))
} else if r.len() == 1 {
Ok(()) Ok(())
} else { } else {
Err(ConsistencyError::UuidNotUnique(uuid.to_string())) Err(ConsistencyError::UuidNotUnique(uuid.to_string()))
} }
}
Err(_) => Err(ConsistencyError::QueryServerSearchFailure),
}
}) })
.filter(|v| v.is_err()) .filter(|v| v.is_err())
.collect() .collect()

4
kanidmd/src/lib/plugins/domain.rs
View file

@ -39,12 +39,12 @@ impl Plugin for Domain {
{ {
// We always set this, because the DB uuid is authorative. // We always set this, because the DB uuid is authorative.
let u = Value::new_uuid(qs.get_domain_uuid()); let u = Value::new_uuid(qs.get_domain_uuid());
e.set_avas("domain_uuid", vec![u]); e.set_ava("domain_uuid", btreeset![u]);
ltrace!(au, "plugin_domain: Applying uuid transform"); ltrace!(au, "plugin_domain: Applying uuid transform");
// We only apply this if one isn't provided. // We only apply this if one isn't provided.
if !e.attribute_pres("domain_name") { if !e.attribute_pres("domain_name") {
let n = Value::new_iname_s("example.com"); let n = Value::new_iname_s("example.com");
e.set_avas("domain_name", vec![n]); e.set_ava("domain_name", btreeset![n]);
ltrace!(au, "plugin_domain: Applying domain_name transform"); ltrace!(au, "plugin_domain: Applying domain_name transform");
} }
ltrace!(au, "{:?}", e); ltrace!(au, "{:?}", e);

2
kanidmd/src/lib/plugins/gidnumber.rs
View file

@ -55,7 +55,7 @@ fn apply_gidnumber<T: Clone>(
let gid_v = Value::new_uint32(gid); let gid_v = Value::new_uint32(gid);
ladmin_info!(au, "Generated {} for {:?}", gid, u_ref); ladmin_info!(au, "Generated {} for {:?}", gid, u_ref);
e.set_avas("gidnumber", vec![gid_v]); e.set_ava("gidnumber", btreeset![gid_v]);
Ok(()) Ok(())
} else if let Some(gid) = e.get_ava_single_uint32("gidnumber") { } else if let Some(gid) = e.get_ava_single_uint32("gidnumber") {
// If they provided us with a gid number, ensure it's in a safe range. // If they provided us with a gid number, ensure it's in a safe range.

679
kanidmd/src/lib/plugins/memberof.rs
View file

@ -13,178 +13,195 @@
use crate::audit::AuditScope; use crate::audit::AuditScope;
use crate::entry::{Entry, EntryCommitted, EntryInvalid, EntrySealed}; use crate::entry::{Entry, EntryCommitted, EntryInvalid, EntrySealed};
use crate::event::{CreateEvent, DeleteEvent, ModifyEvent}; use crate::event::{CreateEvent, DeleteEvent, ModifyEvent};
use crate::modify::{Modify, ModifyList}; // use crate::modify::{Modify, ModifyList};
use crate::plugins::Plugin; use crate::plugins::Plugin;
use crate::server::QueryServerTransaction; use crate::server::QueryServerTransaction;
use crate::server::{QueryServerReadTransaction, QueryServerWriteTransaction}; use crate::server::{QueryServerReadTransaction, QueryServerWriteTransaction};
use crate::value::{PartialValue, Value}; use crate::value::{PartialValue, Value};
use kanidm_proto::v1::{ConsistencyError, OperationError}; use kanidm_proto::v1::{ConsistencyError, OperationError};
use hashbrown::HashMap;
use std::collections::BTreeSet; use std::collections::BTreeSet;
use uuid::Uuid; use uuid::Uuid;
lazy_static! { lazy_static! {
static ref CLASS_GROUP: PartialValue = PartialValue::new_class("group"); static ref CLASS_GROUP: PartialValue = PartialValue::new_class("group");
static ref CLASS_MEMBEROF: Value = Value::new_class("memberof");
} }
pub struct MemberOf; pub struct MemberOf;
fn affected_uuids<'a, STATE>( type EntrySealedCommitted = Entry<EntrySealed, EntryCommitted>;
au: &mut AuditScope, type EntryInvalidCommitted = Entry<EntryInvalid, EntryCommitted>;
changed: Vec<&'a Entry<EntrySealed, STATE>>, type EntryTuple = (EntrySealedCommitted, EntryInvalidCommitted);
) -> Vec<&'a Uuid>
where
STATE: std::fmt::Debug,
{
// From the list of groups which were changed in this operation:
// let changed_groups: Vec<_> = changed
let mut affected_uuids: Vec<&Uuid> = changed
.into_iter()
.filter(|e| e.attribute_value_pres("class", &CLASS_GROUP))
/*
.collect();
ltrace!(au, "groups reporting change: {:?}", changed_groups);
// Now, build a map of all UUID's that will require updates as a result of this change
let mut affected_uuids: Vec<&Uuid> = changed_groups
.iter()
*/
.filter_map(|e| {
// Only groups with member get collected up here.
e.get_ava("member")
})
// Flatten the member's to the list.
.flatten()
.filter_map(|uv| uv.to_ref_uuid())
.collect();
ltrace!(au, "uuids reporting change: {:?}", affected_uuids); fn do_memberof(
// IDEA: promote groups to head of the affected_uuids set!
//
// This isn't worth doing - it's only used in create/delete, it would not
// really make a large performance difference. Better to target improvements
// in the apply_memberof fn.
affected_uuids.sort();
// Remove dups
affected_uuids.dedup();
affected_uuids
}
fn apply_memberof(
au: &mut AuditScope, au: &mut AuditScope,
qs: &QueryServerWriteTransaction, qs: &QueryServerWriteTransaction,
affected_uuids: Vec<&Uuid>, uuid: &Uuid,
tgte: &mut EntryInvalidCommitted,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
ltrace!(au, " => entering apply_memberof"); // search where we are member
ltrace!(au, "affected uuids -> {:?}", affected_uuids);
// Apply member takes a list of changes. We then filter that to only the changed groups
// and using this, we determine a list of UUID's from members that will be required to
// re-examine their MO attributes.
// Given the list of UUID that require changes, we attempt to trigger MO updates on groups
// first to stabilise the MO graph before we start triggering changes on entries.
//
// it's important to note that each change itself, especially groups, could trigger there
// own recursive updates until the base case - stable, no changes - is reached.
//
// That means the termination of recursion is ALWAYS to be found in the post_modify
// callback, as regardless of initial entry point, all subsequent MO internal operations
// are modifies - it is up to post_modify to break cycles!
// Now work on the affected set.
// For each affected uuid
affected_uuids.into_iter().try_for_each(|a_uuid| {
// search where group + Eq("member": "uuid")
let groups = qs let groups = qs
.internal_search( .internal_search(
au, au,
filter!(f_and!([ filter!(f_and!([
f_eq("class", CLASS_GROUP.clone()), f_eq("class", CLASS_GROUP.clone()),
f_eq("member", PartialValue::new_refer_r(a_uuid)) f_eq("member", PartialValue::new_refer(*uuid))
])), ])),
) )
.map_err(|e| { .map_err(|e| {
ladmin_error!(au, "internal search failure -> {:?}", e); ladmin_error!(au, "internal search failure -> {:?}", e);
e e
})?; })?;
// get UUID of all groups + all memberof values
let mut dir_mo_set: Vec<Value> = groups // Ensure we are MO capable.
.iter() tgte.add_ava("class", CLASS_MEMBEROF.clone());
.map(|g| { // Clear the dmo + mos, we will recreate them now.
// These are turned into reference values. // This is how we handle deletes/etc.
Value::new_refer(*g.get_uuid()) tgte.pop_ava("memberof");
tgte.pop_ava("directmemberof");
// Add all the direct mo's and mos.
groups.iter().for_each(|g| {
// TODO: Change add_ava to remove this alloc/clone.
let dmo = Value::new_refer(*g.get_uuid());
tgte.add_ava("directmemberof", dmo.clone());
tgte.add_ava("memberof", dmo);
if let Some(miter) = g.get_ava("memberof") {
miter.for_each(|mo| {
tgte.add_ava("memberof", mo.clone());
}) })
.collect(); };
});
// No need to dedup this. Sorting could be of questionable ltrace!(
// value too though ...
dir_mo_set.sort();
dir_mo_set.dedup();
let mut mo_set: Vec<Value> = groups
.iter()
.map(|g| {
// TODO #61: This could be more effecient
let mut v = vec![Value::new_refer(*g.get_uuid())];
if let Some(mos) = g.get_ava("memberof") {
for mo in mos {
// This is cloning the existing reference values
v.push(mo.clone())
}
}
v
})
.flatten()
.collect();
mo_set.sort();
mo_set.dedup();
ltrace!(au, "Updating {:?} to be dir mo {:?}", a_uuid, dir_mo_set);
ltrace!(au, "Updating {:?} to be mo {:?}", a_uuid, mo_set);
// first add a purged memberof to remove all mo we no longer
// support.
// TODO #61: Could this be more efficient
// TODO #68: Could this affect replication? Or should the CL work out the
// true diff of the operation?
let mo_purge = vec![
Modify::Present("class".to_string(), Value::new_class("memberof")),
Modify::Purged("memberof".to_string()),
Modify::Purged("directmemberof".to_string()),
];
// create modify present memberof all uuids
let mod_set: Vec<_> = mo_purge
.into_iter()
.chain(
mo_set
.into_iter()
.map(|mo_uuid| Modify::Present("memberof".to_string(), mo_uuid)),
)
.chain(
dir_mo_set
.into_iter()
.map(|mo_uuid| Modify::Present("directmemberof".to_string(), mo_uuid)),
)
.collect();
// apply to affected uuid
let modlist = ModifyList::new_list(mod_set);
qs.internal_modify(
au, au,
filter!(f_eq("uuid", PartialValue::new_uuid(*a_uuid))), "Updating {:?} to be dir mo {:?}",
modlist, uuid,
) tgte.get_ava_set("directmemberof")
);
ltrace!(
au,
"Updating {:?} to be mo {:?}",
uuid,
tgte.get_ava_set("memberof")
);
Ok(())
}
#[allow(clippy::cognitive_complexity)]
fn apply_memberof(
au: &mut AuditScope,
qs: &QueryServerWriteTransaction,
// TODO: Experiment with HashSet/BTreeSet here instead of vec.
// May require https://github.com/rust-lang/rust/issues/62924 to allow poping
mut group_affect: Vec<Uuid>,
) -> Result<(), OperationError> {
ltrace!(au, " => entering apply_memberof");
ltrace!(au, " => initial group_affect {:?}", group_affect);
// We can't cache groups, because we need to be continually writing
// and querying them. But we can cache anything we find in the process
// to speed up the later other_affect write op, and we can use this
// to avoid loading things that aren't groups.
// All other changed entries (mo, dmo cleared)
let mut other_cache: HashMap<Uuid, EntryTuple> = HashMap::with_capacity(group_affect.len() * 2);
while !group_affect.is_empty() {
group_affect.sort();
group_affect.dedup();
// Prep the write lists
let mut pre_candidates = Vec::with_capacity(group_affect.len());
let mut candidates = Vec::with_capacity(group_affect.len());
// Ignore recycled/tombstones
let filt = filter!(FC::Or(
group_affect
.drain(0..)
.map(|u| f_eq("uuid", PartialValue::new_uuid(u)))
.collect()
));
let mut work_set = qs.internal_search_writeable(au, filt)?;
// Load the vecdeque with this batch.
while let Some((pre, mut tgte)) = work_set.pop() {
let guuid = *pre.get_uuid();
// load the entry from the db.
if !tgte.attribute_value_pres("class", &CLASS_GROUP) {
// It's not a group, we'll deal with you later. We should NOT
// have seen this UUID before, as either we are on the first
// iteration OR the checks belowe should have filtered it out.
ltrace!(au, "not a group, delaying update to -> {:?}", guuid);
other_cache.insert(guuid, (pre, tgte));
continue;
}
ltrace!(au, "=> processing group update -> {:?}", guuid);
do_memberof(au, qs, &guuid, &mut tgte)?;
// Did we change? Note we don't check if the class changed, only if mo changed.
if pre.get_ava_set("memberof") != tgte.get_ava_set("memberof")
|| pre.get_ava_set("directmemberof") != tgte.get_ava_set("directmemberof")
{
// Yes we changed - we now must process all our members, as they need to
// inherit changes. Some of these members COULD be non groups, but we
// handle that in the dbload step.
ltrace!(
au,
"{:?} changed, flagging members as groups to change. ",
guuid
);
if let Some(miter) = tgte.get_ava_as_refuuid("member") {
group_affect.extend(miter.filter(|m| !other_cache.contains_key(m)));
};
// push the entries to pre/cand
pre_candidates.push(pre);
candidates.push(tgte);
} else {
ltrace!(au, "{:?} stable", guuid);
}
}
debug_assert!(pre_candidates.len() == candidates.len());
// Write this stripe if populated.
if !pre_candidates.is_empty() {
qs.internal_batch_modify(au, pre_candidates, candidates)
.map_err(|e| { .map_err(|e| {
ladmin_error!(au, "Internal modification failure -> {:?}", e); ladmin_error!(au, "Failed to commit memberof group set {:?}", e);
e e
}) })?;
}) }
// Next loop!
}
// ALL GROUP MOS + DMOS ARE NOW STABLE. We can load these into other items directly.
let mut pre_candidates = Vec::with_capacity(other_cache.len());
let mut candidates = Vec::with_capacity(other_cache.len());
other_cache
.into_iter()
.try_for_each(|(auuid, (pre, mut tgte))| {
ltrace!(au, "=> processing affected uuid {:?}", auuid);
debug_assert!(!tgte.attribute_value_pres("class", &CLASS_GROUP));
do_memberof(au, qs, &auuid, &mut tgte)?;
// Only write if a change occured.
if pre.get_ava_set("memberof") != tgte.get_ava_set("memberof")
|| pre.get_ava_set("directmemberof") != tgte.get_ava_set("directmemberof")
{
pre_candidates.push(pre);
candidates.push(tgte);
}
Ok(())
})?;
// Turn the other_cache into a write set.
// Write the batch out in a single stripe.
qs.internal_batch_modify(au, pre_candidates, candidates)
// Done! 🎉
} }
impl Plugin for MemberOf { impl Plugin for MemberOf {
@ -192,20 +209,31 @@ impl Plugin for MemberOf {
"memberof" "memberof"
} }
// TODO #61: We could make this more effecient by limiting change detection to ONLY member/memberof
// attrs rather than any attrs.
fn post_create( fn post_create(
au: &mut AuditScope, au: &mut AuditScope,
qs: &QueryServerWriteTransaction, qs: &QueryServerWriteTransaction,
cand: &[Entry<EntrySealed, EntryCommitted>], cand: &[Entry<EntrySealed, EntryCommitted>],
_ce: &CreateEvent, _ce: &CreateEvent,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
// let group_affect = cand
// Trigger apply_memberof on all because they changed. .iter()
let cand_refs: Vec<&Entry<_, _>> = cand.iter().map(|e| e).collect(); .map(|e| e.get_uuid())
let uuids = affected_uuids(au, cand_refs); .chain(
apply_memberof(au, qs, uuids) cand.iter()
.filter_map(|e| {
// Is it a group?
if e.attribute_value_pres("class", &CLASS_GROUP) {
e.get_ava_as_refuuid("member")
} else {
None
}
})
.flatten(),
)
.copied()
.collect();
apply_memberof(au, qs, group_affect)
} }
fn post_modify( fn post_modify(
@ -215,40 +243,37 @@ impl Plugin for MemberOf {
cand: &[Entry<EntrySealed, EntryCommitted>], cand: &[Entry<EntrySealed, EntryCommitted>],
_me: &ModifyEvent, _me: &ModifyEvent,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
// The condition here is critical - ONLY trigger on entries where changes occur! // TODO: Limit this to when it's a class, member, mo, dmo change instead.
let mut changed: Vec<&Uuid> = pre_cand let group_affect = cand
.iter() .iter()
.zip(cand.iter()) .map(|post| post.get_uuid())
.filter(|(pre, post)| { .chain(
// This is the base case to break cycles in recursion! pre_cand
( .iter()
// If it was a group, or will become a group. .filter_map(|pre| {
post.attribute_value_pres("class", &CLASS_GROUP) if pre.attribute_value_pres("class", &CLASS_GROUP) {
|| pre.attribute_value_pres("class", &CLASS_GROUP) pre.get_ava_as_refuuid("member")
} else {
None
}
})
.flatten(),
) )
// And the group has changed ... .chain(
&& pre != post cand.iter()
// Then memberof should be updated! .filter_map(|post| {
if post.attribute_value_pres("class", &CLASS_GROUP) {
post.get_ava_as_refuuid("member")
} else {
None
}
}) })
// Flatten the pre-post tuples. We no longer care if it was .flatten(),
// pre-post )
.flat_map(|(pre, post)| vec![pre, post]) .copied()
.inspect(|e| {
ltrace!(au, "group reporting change: {:?}", e);
})
.filter_map(|e| {
// Only groups with member get collected up here.
e.get_ava_as_refuuid("member")
})
// Flatten the uuid reference lists.
.flatten()
.collect(); .collect();
// Now tidy them up to reduce excesse searches/work. apply_memberof(au, qs, group_affect)
changed.sort();
changed.dedup();
apply_memberof(au, qs, changed)
} }
fn pre_delete( fn pre_delete(
@ -265,15 +290,8 @@ impl Plugin for MemberOf {
// As a result, on restore, the graph of where it was a member // As a result, on restore, the graph of where it was a member
// would have to be rebuilt. // would have to be rebuilt.
// //
// AN interesting possibility could be NOT to purge MO on delete
// and use that to rebuild the forward graph of member -> item, but
// due to the nature of MO, we do not know the difference between
// direct and indirect membership, meaning we would be safer
// to not do this.
// NOTE: DO NOT purge directmemberof - we use that to restore memberships // NOTE: DO NOT purge directmemberof - we use that to restore memberships
// in recycle revive! // in recycle revive!
cand.iter_mut().for_each(|e| e.purge_ava("memberof")); cand.iter_mut().for_each(|e| e.purge_ava("memberof"));
Ok(()) Ok(())
} }
@ -284,11 +302,23 @@ impl Plugin for MemberOf {
cand: &[Entry<EntrySealed, EntryCommitted>], cand: &[Entry<EntrySealed, EntryCommitted>],
_ce: &DeleteEvent, _ce: &DeleteEvent,
) -> Result<(), OperationError> { ) -> Result<(), OperationError> {
// // Similar condition to create - we only trigger updates on groups's members,
// Trigger apply_memberof on all - because they all changed. // so that they can find they are no longer a mo of what was deleted.
let cand_refs: Vec<&Entry<_, _>> = cand.iter().map(|e| e).collect(); let group_affect = cand
let uuids = affected_uuids(au, cand_refs); .iter()
apply_memberof(au, qs, uuids) .filter_map(|e| {
// Is it a group?
if e.attribute_value_pres("class", &CLASS_GROUP) {
e.get_ava_as_refuuid("member")
} else {
None
}
})
.flatten()
.copied()
.collect();
apply_memberof(au, qs, group_affect)
} }
fn verify( fn verify(
@ -309,9 +339,6 @@ impl Plugin for MemberOf {
// for each entry in the DB (live). // for each entry in the DB (live).
for e in all_cand { for e in all_cand {
// create new map
// let mo_set: BTreeMap<String, ()> = BTreeMap::new();
// searcch direct memberships of live groups.
let filt_in = filter!(f_and!([ let filt_in = filter!(f_and!([
f_eq("class", PartialValue::new_class("group")), f_eq("class", PartialValue::new_class("group")),
f_eq("member", PartialValue::new_refer(*e.get_uuid())) f_eq("member", PartialValue::new_refer(*e.get_uuid()))
@ -326,8 +353,16 @@ impl Plugin for MemberOf {
}; };
// for all direct -> add uuid to map // for all direct -> add uuid to map
let d_groups_set: BTreeSet<&Uuid> = let d_groups_set: BTreeSet<_> = direct_memberof
direct_memberof.iter().map(|e| e.get_uuid()).collect(); .iter()
.map(|e| Value::new_refer(*e.get_uuid()))
.collect();
let d_groups_set = if d_groups_set.is_empty() {
None
} else {
Some(d_groups_set)
};
ltrace!( ltrace!(
au, au,
@ -336,40 +371,32 @@ impl Plugin for MemberOf {
d_groups_set d_groups_set
); );
let dmos: Vec<&Uuid> = match e.get_ava_as_refuuid("directmemberof") { match (e.get_ava_set("directmemberof"), d_groups_set) {
// Avoid a reference issue to return empty set (Some(edmos), Some(dmos)) => {
Some(dmos) => dmos.collect(), let diff: Vec<_> = dmos.symmetric_difference(edmos).collect();
// No memberof, return empty set. if !diff.is_empty() {
None => Vec::new(), ladmin_error!(
}; au,
"MemberOfInvalid: Entry {}, DMO has inconsistencies -> {:?}",
ltrace!(au, "Direct Member Of Set {:?} -> {:?}", e.get_uuid(), dmos); e,
diff
if dmos.len() != direct_memberof.len() { );
r.push(Err(ConsistencyError::MemberOfInvalid(e.get_id())));
}
}
(None, None) => {
// Ok
}
_ => {
ladmin_error!( ladmin_error!(
au, au,
"MemberOfInvalid directmemberof set and DMO search set differ in size: {}", "MemberOfInvalid directmemberof set and DMO search set differ in size: {}",
e e.get_uuid()
); );
r.push(Err(ConsistencyError::MemberOfInvalid(e.get_id()))); r.push(Err(ConsistencyError::MemberOfInvalid(e.get_id())));
debug_assert!(false); }
// Next entry }
continue;
};
for mo_uuid in dmos {
if !d_groups_set.contains(mo_uuid) {
ladmin_error!(
au,
"MemberOfInvalid: Entry {}, MO {:?} not in direct groups",
e,
mo_uuid
);
r.push(Err(ConsistencyError::MemberOfInvalid(e.get_id())));
// Next entry
continue;
}
}
// Could check all dmos in mos? // Could check all dmos in mos?
/* To check nested! */ /* To check nested! */
@ -507,7 +534,7 @@ mod tests {
let eb: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EB); let eb: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EB);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
let preload = Vec::new(); let preload = Vec::new();
let create = vec![ea, eb]; let create = vec![ea, eb];
@ -537,8 +564,8 @@ mod tests {
let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
let preload = Vec::new(); let preload = Vec::new();
let create = vec![ea, eb, ec]; let create = vec![ea, eb, ec];
@ -588,9 +615,9 @@ mod tests {
let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
let preload = Vec::new(); let preload = Vec::new();
let create = vec![ea, eb, ec]; let create = vec![ea, eb, ec];
@ -642,13 +669,13 @@ mod tests {
let mut ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED); let mut ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_D).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_D).unwrap());
ed.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ed.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
let preload = Vec::new(); let preload = Vec::new();
let create = vec![ea, eb, ec, ed]; let create = vec![ea, eb, ec, ed];
@ -745,7 +772,7 @@ mod tests {
let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_modify_test!( run_modify_test!(
@ -798,7 +825,7 @@ mod tests {
let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_modify_test!( run_modify_test!(
@ -853,8 +880,8 @@ mod tests {
let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_modify_test!( run_modify_test!(
@ -915,9 +942,9 @@ mod tests {
let ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED); let ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_D).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_D).unwrap());
let preload = vec![ea, eb, ec, ed]; let preload = vec![ea, eb, ec, ed];
run_modify_test!( run_modify_test!(
@ -987,8 +1014,8 @@ mod tests {
let mut eb: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EB); let mut eb: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EB);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
let preload = vec![ea, eb]; let preload = vec![ea, eb];
run_modify_test!( run_modify_test!(
@ -1023,10 +1050,10 @@ mod tests {
let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_modify_test!( run_modify_test!(
@ -1079,11 +1106,11 @@ mod tests {
let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_modify_test!( run_modify_test!(
@ -1137,20 +1164,20 @@ mod tests {
let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_modify_test!( run_modify_test!(
@ -1212,30 +1239,30 @@ mod tests {
let mut ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED); let mut ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_D).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_D).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ed.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ed.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
let preload = vec![ea, eb, ec, ed]; let preload = vec![ea, eb, ec, ed];
run_modify_test!( run_modify_test!(
@ -1306,8 +1333,8 @@ mod tests {
let mut eb: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EB); let mut eb: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EB);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
let preload = vec![ea, eb]; let preload = vec![ea, eb];
run_delete_test!( run_delete_test!(
@ -1336,12 +1363,12 @@ mod tests {
let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_delete_test!( run_delete_test!(
@ -1380,12 +1407,12 @@ mod tests {
let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_delete_test!( run_delete_test!(
@ -1425,20 +1452,20 @@ mod tests {
let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC); let mut ec: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(EC);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
let preload = vec![ea, eb, ec]; let preload = vec![ea, eb, ec];
run_delete_test!( run_delete_test!(
@ -1481,30 +1508,30 @@ mod tests {
let mut ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED); let mut ed: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(ED);
ea.add_ava("member", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("member", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ea.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); ea.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
eb.add_ava("member", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("member", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
eb.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); eb.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("member", &Value::new_refer_s(&UUID_D).unwrap()); ec.add_ava("member", Value::new_refer_s(&UUID_D).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ec.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); ec.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
ed.add_ava("member", &Value::new_refer_s(&UUID_A).unwrap()); ed.add_ava("member", Value::new_refer_s(&UUID_A).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_A).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_A).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_B).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_B).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_C).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_C).unwrap());
ed.add_ava("memberof", &Value::new_refer_s(&UUID_D).unwrap()); ed.add_ava("memberof", Value::new_refer_s(&UUID_D).unwrap());
let preload = vec![ea, eb, ec, ed]; let preload = vec![ea, eb, ec, ed];
run_delete_test!( run_delete_test!(

16
kanidmd/src/lib/plugins/password_import.rs
View file

@ -57,8 +57,8 @@ impl Plugin for PasswordImport {
None => { None => {
// just set it then! // just set it then!
let c = Credential::new_from_password(pw); let c = Credential::new_from_password(pw);
e.set_avas("primary_credential", e.set_ava("primary_credential",
vec![Value::new_credential("primary", c)]); btreeset![Value::new_credential("primary", c)]);
Ok(()) Ok(())
} }
} }
@ -105,18 +105,18 @@ impl Plugin for PasswordImport {
Some(c) => { Some(c) => {
// This is the major diff to create, we can update in place! // This is the major diff to create, we can update in place!
let c = c.update_password(pw); let c = c.update_password(pw);
e.set_avas( e.set_ava(
"primary_credential", "primary_credential",
vec![Value::new_credential("primary", c)], btreeset![Value::new_credential("primary", c)],
); );
Ok(()) Ok(())
} }
None => { None => {
// just set it then! // just set it then!
let c = Credential::new_from_password(pw); let c = Credential::new_from_password(pw);
e.set_avas( e.set_ava(
"primary_credential", "primary_credential",
vec![Value::new_credential("primary", c)], btreeset![Value::new_credential("primary", c)],
); );
Ok(()) Ok(())
} }
@ -207,7 +207,7 @@ mod tests {
); );
let c = Credential::new_password_only("password"); let c = Credential::new_password_only("password");
ea.add_ava("primary_credential", &Value::new_credential("primary", c)); ea.add_ava("primary_credential", Value::new_credential("primary", c));
let preload = vec![ea]; let preload = vec![ea];
@ -241,7 +241,7 @@ mod tests {
let totp = TOTP::generate_secure("test_totp".to_string(), TOTP_DEFAULT_STEP); let totp = TOTP::generate_secure("test_totp".to_string(), TOTP_DEFAULT_STEP);
let c = Credential::new_password_only("password").update_totp(totp); let c = Credential::new_password_only("password").update_totp(totp);
ea.add_ava("primary_credential", &Value::new_credential("primary", c)); ea.add_ava("primary_credential", Value::new_credential("primary", c));
let preload = vec![ea]; let preload = vec![ea];

4
kanidmd/src/lib/plugins/protected.rs
View file

@ -8,8 +8,8 @@ use crate::event::{CreateEvent, DeleteEvent, ModifyEvent};
use crate::modify::Modify; use crate::modify::Modify;
use crate::server::QueryServerWriteTransaction; use crate::server::QueryServerWriteTransaction;
use crate::value::{PartialValue, Value}; use crate::value::{PartialValue, Value};
use hashbrown::HashSet;
use kanidm_proto::v1::OperationError; use kanidm_proto::v1::OperationError;
use std::collections::HashSet;
pub struct Protected {} pub struct Protected {}
@ -19,7 +19,7 @@ pub struct Protected {}
lazy_static! { lazy_static! {
static ref ALLOWED_ATTRS: HashSet<&'static str> = { static ref ALLOWED_ATTRS: HashSet<&'static str> = {
let mut m = HashSet::new(); let mut m = HashSet::with_capacity(8);
// Allow modification of some schema class types to allow local extension // Allow modification of some schema class types to allow local extension
// of schema types. // of schema types.
m.insert("must"); m.insert("must");

179
kanidmd/src/lib/plugins/refint.rs
View file

@ -9,12 +9,14 @@
// when that is written, as they *both* manipulate and alter entry reference // when that is written, as they *both* manipulate and alter entry reference
// data, so we should be careful not to step on each other. // data, so we should be careful not to step on each other.
use std::collections::HashSet as Set; use hashbrown::HashSet as Set;
use std::collections::BTreeSet;
use crate::audit::AuditScope; use crate::audit::AuditScope;
use crate::entry::{Entry, EntryCommitted, EntrySealed}; use crate::entry::{Entry, EntryCommitted, EntrySealed};
use crate::event::{CreateEvent, DeleteEvent, ModifyEvent}; use crate::event::{CreateEvent, DeleteEvent, ModifyEvent};
use crate::modify::{Modify, ModifyInvalid, ModifyList}; use crate::filter::f_eq;
use crate::modify::Modify;
use crate::plugins::Plugin; use crate::plugins::Plugin;
use crate::schema::SchemaTransaction; use crate::schema::SchemaTransaction;
use crate::server::QueryServerTransaction; use crate::server::QueryServerTransaction;
@ -28,32 +30,52 @@ use uuid::Uuid;
pub struct ReferentialIntegrity; pub struct ReferentialIntegrity;
impl ReferentialIntegrity { impl ReferentialIntegrity {
fn check_uuid_exists( fn check_uuids_exist<'a, I>(
au: &mut AuditScope, au: &mut AuditScope,
qs: &QueryServerWriteTransaction, qs: &QueryServerWriteTransaction,
rtype: &str, ref_iter: I,
uuid_value: &Value, ) -> Result<(), OperationError>
) -> Result<(), OperationError> { where
let uuid = uuid_value.to_ref_uuid().ok_or_else(|| { I: Iterator<Item = &'a Value>,
ladmin_error!(au, "uuid value could not convert to reference uuid"); {
OperationError::InvalidAttribute("uuid could not become reference value".to_string()) let inner: Result<Vec<_>, _> = ref_iter
})?; .map(|uuid_value| {
// NOTE: This only checks LIVE entries (not using filter_all) uuid_value
let filt_in = filter!(f_eq("uuid", PartialValue::new_uuid(*uuid))); .to_ref_uuid()
.map(|uuid| f_eq("uuid", PartialValue::new_uuid(*uuid)))
.ok_or_else(|| {
ladmin_error!(au, "ref value could not convert to reference uuid");
OperationError::InvalidAttribute(
"uuid could not become reference value".to_string(),
)
})
})
.collect();
let inner = inner?;
if inner.is_empty() {
// There is nothing to check! Move on.
ladmin_info!(au, "no reference types modified, skipping check");
return Ok(());
}
// F_inc(lusion). All items of inner must be 1 or more, or the filter
// will fail. This will return the union of the inclusion after the
// operationn.
let filt_in = filter!(f_inc(inner));
let b = qs.internal_exists(au, filt_in).map_err(|e| { let b = qs.internal_exists(au, filt_in).map_err(|e| {
ladmin_error!(au, "internal exists failure -> {:?}", e); ladmin_error!(au, "internal exists failure -> {:?}", e);
e e
})?; })?;
// Is the reference in the result set? // Is the existance of all id's confirmed?
if b { if b {
Ok(()) Ok(())
} else { } else {
ladmin_error!( ladmin_error!(
au, au,
"{:?}:{:?} UUID reference not found in database", "UUID reference set size differs from query result size <fast path, no uuid info available>"
rtype,
uuid
); );
Err(OperationError::Plugin(PluginError::ReferentialIntegrity( Err(OperationError::Plugin(PluginError::ReferentialIntegrity(
"Uuid referenced not found in database".to_string(), "Uuid referenced not found in database".to_string(),
@ -90,20 +112,18 @@ impl Plugin for ReferentialIntegrity {
let schema = qs.get_schema(); let schema = qs.get_schema();
let ref_types = schema.get_reference_types(); let ref_types = schema.get_reference_types();
// For all cands // Fast Path
for c in cand { let i = cand
// For all reference in each cand. .iter()
for rtype in ref_types.values() { .map(|c| {
// If the attribute is present ref_types
if let Some(vs) = c.get_ava(&rtype.name) { .values()
// For each value in the set. .filter_map(move |rtype| c.get_ava(&rtype.name))
for v in vs { })
Self::check_uuid_exists(au, qs, &rtype.name, v)? .flatten()
} .flatten();
}
} Self::check_uuids_exist(au, qs, i)
}
Ok(())
} }
fn post_modify( fn post_modify(
@ -116,17 +136,19 @@ impl Plugin for ReferentialIntegrity {
let schema = qs.get_schema(); let schema = qs.get_schema();
let ref_types = schema.get_reference_types(); let ref_types = schema.get_reference_types();
// For all mods let i = me.modlist.into_iter().filter_map(|modify| {
for modify in me.modlist.into_iter() {
// If the mod affects a reference type and being ADDED.
if let Modify::Present(a, v) = &modify { if let Modify::Present(a, v) = &modify {
if let Some(a_type) = ref_types.get(a) { if ref_types.get(a).is_some() {
// So it is a reference type, now check it. Some(v)
Self::check_uuid_exists(au, qs, &a_type.name, v)? } else {
None
} }
} else {
None
} }
} });
Ok(())
Self::check_uuids_exist(au, qs, i)
} }
fn post_delete( fn post_delete(
@ -163,27 +185,24 @@ impl Plugin for ReferentialIntegrity {
ltrace!(au, "refint post_delete filter {:?}", filt); ltrace!(au, "refint post_delete filter {:?}", filt);
// Create a modlist: let removed_ids: BTreeSet<_> = cand
// In each, create a "removed" for each attr:uuid pair .iter()
let modlist: ModifyList<ModifyInvalid> = ModifyList::new_list( .map(|e| PartialValue::new_refer(*e.get_uuid()))
// uuids .collect();
// .iter()
cand.iter() let work_set = qs.internal_search_writeable(au, filt)?;
.map(|e| e.get_uuid())
.map(|u| { let (pre_candidates, candidates) = work_set
ref_types.values().map(move |r_type| { .into_iter()
Modify::Removed(r_type.name.clone(), PartialValue::new_refer(*u)) .map(|(pre, mut post)| {
ref_types
.values()
.for_each(|attr| post.remove_avas(attr.name.as_str(), &removed_ids));
(pre, post)
}) })
}) .unzip();
.flatten()
.collect(),
);
ltrace!(au, "refint post_delete modlist {:?}", modlist); qs.internal_batch_modify(au, pre_candidates, candidates)
// Do an internal modify to apply the modlist and filter.
qs.internal_modify(au, filt, modlist)
} }
fn verify( fn verify(
@ -239,6 +258,7 @@ impl Plugin for ReferentialIntegrity {
mod tests { mod tests {
// #[macro_use] // #[macro_use]
// use crate::plugins::Plugin; // use crate::plugins::Plugin;
use crate::constants::UUID_DOES_NOT_EXIST;
use crate::entry::{Entry, EntryInit, EntryNew}; use crate::entry::{Entry, EntryInit, EntryNew};
use crate::modify::{Modify, ModifyList}; use crate::modify::{Modify, ModifyList};
use crate::server::{QueryServerTransaction, QueryServerWriteTransaction}; use crate::server::{QueryServerTransaction, QueryServerWriteTransaction};
@ -422,6 +442,51 @@ mod tests {
); );
} }
// Check that even when SOME references exist, so long as one does not,
// we fail.
#[test]
fn test_modify_uuid_reference_partial_not_exist() {
let ea: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(
r#"{
"attrs": {
"class": ["group"],
"name": ["testgroup_a"],
"description": ["testgroup"],
"uuid": ["d2b496bd-8493-47b7-8142-f568b5cf47ee"]
}
}"#,
);
let eb: Entry<EntryInit, EntryNew> = Entry::unsafe_from_entry_str(
r#"{
"attrs": {
"class": ["group"],
"name": ["testgroup_b"],
"description": ["testgroup"]
}
}"#,
);
let preload = vec![ea, eb];
run_modify_test!(
Err(OperationError::Plugin(PluginError::ReferentialIntegrity(
"Uuid referenced not found in database".to_string()
))),
preload,
filter!(f_eq("name", PartialValue::new_iname("testgroup_b"))),
ModifyList::new_list(vec![
Modify::Present(
"member".to_string(),
Value::new_refer_s("d2b496bd-8493-47b7-8142-f568b5cf47ee").unwrap()
),
Modify::Present("member".to_string(), Value::new_refer(*UUID_DOES_NOT_EXIST)),
]),
None,
|_, _| {}
);
}
// Modify removes the reference to an entry // Modify removes the reference to an entry
#[test] #[test]
fn test_modify_remove_referee() { fn test_modify_remove_referee() {

6
kanidmd/src/lib/plugins/spn.rs
View file

@ -100,7 +100,7 @@ impl Plugin for Spn {
e e
})?; })?;
ltrace!(au, "plugin_spn: set spn to {:?}", spn); ltrace!(au, "plugin_spn: set spn to {:?}", spn);
e.set_avas("spn", vec![spn]); e.set_ava("spn", btreeset![spn]);
} }
} }
Ok(()) Ok(())
@ -141,7 +141,7 @@ impl Plugin for Spn {
e e
})?; })?;
ltrace!(au, "plugin_spn: set spn to {:?}", spn); ltrace!(au, "plugin_spn: set spn to {:?}", spn);
e.set_avas("spn", vec![spn]); e.set_ava("spn", btreeset![spn]);
} }
} }
Ok(()) Ok(())
@ -255,8 +255,6 @@ impl Plugin for Spn {
); );
debug_assert!(false); debug_assert!(false);
r.push(Err(ConsistencyError::InvalidSPN(e.get_id()))) r.push(Err(ConsistencyError::InvalidSPN(e.get_id())))
} else {
ltrace!(au, "spn is ok! 👍");
} }
} }
None => { None => {

2
kanidmd/src/lib/repl/cid.rs
View file

@ -2,7 +2,7 @@ use kanidm_proto::v1::OperationError;
use std::time::Duration; use std::time::Duration;
use uuid::Uuid; use uuid::Uuid;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Eq, PartialOrd, Ord)] #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Eq, PartialOrd, Ord, Hash)]
pub struct Cid { pub struct Cid {
// Mental note: Derive ord always checks in order of struct fields. // Mental note: Derive ord always checks in order of struct fields.
pub ts: Duration, pub ts: Duration,

10
kanidmd/src/lib/schema.rs
View file

@ -23,10 +23,10 @@ use crate::entry::{Entry, EntryCommitted, EntryInit, EntryNew, EntrySealed};
use crate::value::{IndexType, PartialValue, SyntaxType, Value}; use crate::value::{IndexType, PartialValue, SyntaxType, Value};
use kanidm_proto::v1::{ConsistencyError, OperationError, SchemaError}; use kanidm_proto::v1::{ConsistencyError, OperationError, SchemaError};
use hashbrown::HashMap;
use hashbrown::HashSet;
use std::borrow::Borrow; use std::borrow::Borrow;
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::collections::HashMap;
use std::collections::HashSet;
use uuid::Uuid; use uuid::Uuid;
use concread::cowcell::*; use concread::cowcell::*;
@ -1448,10 +1448,10 @@ impl SchemaTransaction for SchemaReadTransaction {
impl Schema { impl Schema {
pub fn new(audit: &mut AuditScope) -> Result<Self, OperationError> { pub fn new(audit: &mut AuditScope) -> Result<Self, OperationError> {
let s = Schema { let s = Schema {
classes: CowCell::new(HashMap::new()), classes: CowCell::new(HashMap::with_capacity(128)),
attributes: CowCell::new(HashMap::new()), attributes: CowCell::new(HashMap::with_capacity(128)),
unique_cache: CowCell::new(Vec::new()), unique_cache: CowCell::new(Vec::new()),
ref_cache: CowCell::new(HashMap::new()), ref_cache: CowCell::new(HashMap::with_capacity(64)),
}; };
let mut sw = s.write(); let mut sw = s.write();
let r1 = sw.generate_in_memory(audit); let r1 = sw.generate_in_memory(audit);

143
kanidmd/src/lib/server.rs
View file

@ -4,8 +4,9 @@
// This is really only used for long lived, high level types that need clone // This is really only used for long lived, high level types that need clone
// that otherwise can't be cloned. Think Mutex. // that otherwise can't be cloned. Think Mutex.
// use actix::prelude::*; // use actix::prelude::*;
use hashbrown::HashMap;
use std::cell::Cell; use std::cell::Cell;
use std::collections::{BTreeMap, BTreeSet}; use std::collections::BTreeSet;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use uuid::Uuid; use uuid::Uuid;
@ -38,6 +39,10 @@ use crate::schema::{
use crate::value::{PartialValue, SyntaxType, Value}; use crate::value::{PartialValue, SyntaxType, Value};
use kanidm_proto::v1::{ConsistencyError, OperationError, SchemaError}; use kanidm_proto::v1::{ConsistencyError, OperationError, SchemaError};
type EntrySealedCommitted = Entry<EntrySealed, EntryCommitted>;
type EntryInvalidCommitted = Entry<EntryInvalid, EntryCommitted>;
type EntryTuple = (EntrySealedCommitted, EntryInvalidCommitted);
lazy_static! { lazy_static! {
static ref PVCLASS_ATTRIBUTETYPE: PartialValue = PartialValue::new_class("attributetype"); static ref PVCLASS_ATTRIBUTETYPE: PartialValue = PartialValue::new_class("attributetype");
static ref PVCLASS_CLASSTYPE: PartialValue = PartialValue::new_class("classtype"); static ref PVCLASS_CLASSTYPE: PartialValue = PartialValue::new_class("classtype");
@ -1225,7 +1230,8 @@ impl<'a> QueryServerWriteTransaction<'a> {
let revive_cands = let revive_cands =
self.impersonate_search_valid(au, re.filter.clone(), re.filter.clone(), &re.event)?; self.impersonate_search_valid(au, re.filter.clone(), re.filter.clone(), &re.event)?;
let mut dm_mods: BTreeMap<Uuid, ModifyList<ModifyInvalid>> = BTreeMap::new(); let mut dm_mods: HashMap<Uuid, ModifyList<ModifyInvalid>> =
HashMap::with_capacity(revive_cands.len());
revive_cands.into_iter().for_each(|e| { revive_cands.into_iter().for_each(|e| {
// Get this entries uuid. // Get this entries uuid.
@ -1448,6 +1454,127 @@ impl<'a> QueryServerWriteTransaction<'a> {
}) })
} }
/// Used in conjunction with internal_batch_modify, to get a pre/post
/// pair, where post is pre-configured with metadata to allow
/// modificiation before submit back to internal_batch_modify
pub(crate) fn internal_search_writeable(
&self,
audit: &mut AuditScope,
filter: Filter<FilterInvalid>,
) -> Result<Vec<EntryTuple>, OperationError> {
lperf_segment!(audit, "server::internal_search_writeable", || {
let f_valid = filter
.validate(self.get_schema())
.map_err(OperationError::SchemaViolation)?;
let se = SearchEvent::new_internal(f_valid);
self.search(audit, &se).map(|vs| {
vs.into_iter()
.map(|e| {
let writeable = e.clone().invalidate(self.cid.clone());
(e, writeable)
})
.collect()
})
})
}
/// Allows writing batches of modified entries without going through
/// the modlist path. This allows more effecient batch transformations
/// such as memberof, but at the expense that YOU must guarantee you
/// uphold all other plugin and state rules that are important. You
/// probably want modify instead.
pub(crate) fn internal_batch_modify(
&self,
au: &mut AuditScope,
pre_candidates: Vec<Entry<EntrySealed, EntryCommitted>>,
candidates: Vec<Entry<EntryInvalid, EntryCommitted>>,
) -> Result<(), OperationError> {
lperf_segment!(au, "server::internal_batch_modify", || {
lsecurity!(au, "modify initiator: -> internal batch modify");
if pre_candidates.is_empty() && candidates.is_empty() {
// No action needed.
return Ok(());
}
if pre_candidates.len() != candidates.len() {
ladmin_error!(au, "internal_batch_modify - cand lengths differ");
return Err(OperationError::InvalidRequestState);
}
let res: Result<Vec<Entry<EntrySealed, EntryCommitted>>, OperationError> = candidates
.into_iter()
.map(|e| {
e.validate(&self.schema)
.map_err(|e| {
ladmin_error!(au, "Schema Violation {:?}", e);
OperationError::SchemaViolation(e)
})
.map(|e| e.seal())
})
.collect();
let norm_cand: Vec<Entry<_, _>> = res?;
if cfg!(debug_assertions) {
pre_candidates
.iter()
.zip(norm_cand.iter())
.try_for_each(|(pre, post)| {
if pre.get_uuid() == post.get_uuid() {
Ok(())
} else {
ladmin_error!(au, "modify - cand sets not correctly aligned");
Err(OperationError::InvalidRequestState)
}
})?;
}
// Backend Modify
self.be_txn
.modify(au, &pre_candidates, &norm_cand)
.map_err(|e| {
ladmin_error!(au, "Modify operation failed (backend), {:?}", e);
e
})?;
let _ =
self.changed_schema
.replace(norm_cand.iter().chain(pre_candidates.iter()).fold(
false,
|acc, e| {
if acc {
acc
} else {
e.attribute_value_pres("class", &PVCLASS_CLASSTYPE)
|| e.attribute_value_pres("class", &PVCLASS_ATTRIBUTETYPE)
}
},
));
let _ =
self.changed_acp
.replace(norm_cand.iter().chain(pre_candidates.iter()).fold(
false,
|acc, e| {
if acc {
acc
} else {
e.attribute_value_pres("class", &PVCLASS_ACP)
}
},
));
ltrace!(
au,
"Schema reload: {:?}, ACP reload: {:?}",
self.changed_schema,
self.changed_acp
);
ltrace!(au, "Modify operation success");
Ok(())
})
}
/// Migrate 2 to 3 changes the name, domain_name types from iutf8 to iname. /// Migrate 2 to 3 changes the name, domain_name types from iutf8 to iname.
pub fn migrate_2_to_3(&self, au: &mut AuditScope) -> Result<(), OperationError> { pub fn migrate_2_to_3(&self, au: &mut AuditScope) -> Result<(), OperationError> {
lperf_segment!(au, "server::migrate_2_to_3", || { lperf_segment!(au, "server::migrate_2_to_3", || {
@ -3423,9 +3550,9 @@ mod tests {
} }
}"#, }"#,
); );
e1.add_ava("uuid", &Value::new_uuids(uuid).unwrap()); e1.add_ava("uuid", Value::new_uuids(uuid).unwrap());
e1.add_ava("name", &Value::new_iname_s(name)); e1.add_ava("name", Value::new_iname_s(name));
e1.add_ava("displayname", &Value::new_utf8s(name)); e1.add_ava("displayname", Value::new_utf8s(name));
e1 e1
} }
@ -3438,11 +3565,11 @@ mod tests {
} }
}"#, }"#,
); );
e1.add_ava("name", &Value::new_iname_s(name)); e1.add_ava("name", Value::new_iname_s(name));
e1.add_ava("uuid", &Value::new_uuids(uuid).unwrap()); e1.add_ava("uuid", Value::new_uuids(uuid).unwrap());
members members
.iter() .iter()
.for_each(|m| e1.add_ava("member", &Value::new_refer_s(m).unwrap())); .for_each(|m| e1.add_ava("member", Value::new_refer_s(m).unwrap()));
e1 e1
} }

4
kanidmd/src/lib/value.rs
View file

@ -98,7 +98,7 @@ impl fmt::Display for IndexType {
} }
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] #[derive(Hash, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
pub enum SyntaxType { pub enum SyntaxType {
// We need an insensitive string type too ... // We need an insensitive string type too ...
// We also need to "self host" a syntax type, and index type // We also need to "self host" a syntax type, and index type
@ -240,7 +240,7 @@ impl std::fmt::Debug for DataValue {
} }
} }
#[derive(Debug, Clone, Eq, Ord, PartialOrd, PartialEq, Deserialize, Serialize)] #[derive(Hash, Debug, Clone, Eq, Ord, PartialOrd, PartialEq, Deserialize, Serialize)]
pub enum PartialValue { pub enum PartialValue {
Utf8(String), Utf8(String),
Iutf8(String), Iutf8(String),