468 valueset abstraction (#538)

This commit is contained in:
Firstyear 2021-07-30 09:45:25 +10:00 committed by GitHub
parent b8c33ea3ac
commit 27b7572842
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 317 additions and 71 deletions

View file

@ -33,7 +33,7 @@ because names can take many forms such as.
* lastname firstname
And many many more that are not listed here. This is why our names are displayName as a freetext
UTF8 field, with case sensitivitiy and no limits.
UTF8 field, with case sensitivity and no limits.
## Informed consent and Privacy of their data

View file

@ -33,6 +33,7 @@ use crate::repl::cid::Cid;
use crate::schema::{SchemaAttribute, SchemaClass, SchemaTransaction};
use crate::value::{IndexType, SyntaxType};
use crate::value::{PartialValue, Value};
use crate::valueset::ValueSet;
use kanidm_proto::v1::Entry as ProtoEntry;
use kanidm_proto::v1::Filter as ProtoFilter;
use kanidm_proto::v1::{OperationError, SchemaError};
@ -151,7 +152,7 @@ pub struct EntryReduced {
uuid: Uuid,
}
fn compare_attrs(left: &Map<AttrString, Set<Value>>, right: &Map<AttrString, Set<Value>>) -> bool {
fn compare_attrs(left: &Map<AttrString, ValueSet>, right: &Map<AttrString, ValueSet>) -> bool {
// We can't shortcut based on len because cid mod may not be present.
// Build the set of all keys between both.
let allkeys: Set<&str> = left
@ -198,7 +199,7 @@ pub struct Entry<VALID, STATE> {
valid: VALID,
state: STATE,
// We may need to change this to Set to allow borrow of Value -> PartialValue for lookups.
attrs: Map<AttrString, Set<Value>>,
attrs: Map<AttrString, ValueSet>,
}
impl<VALID, STATE> std::fmt::Debug for Entry<VALID, STATE>
@ -270,12 +271,12 @@ impl Entry<EntryInit, EntryNew> {
// Somehow we need to take the tree of e attrs, and convert
// all ref types to our types ...
let map2: Result<Map<AttrString, Set<Value>>, OperationError> = e
let map2: Result<Map<AttrString, ValueSet>, OperationError> = e
.attrs
.iter()
.map(|(k, v)| {
let nk = qs.get_schema().normalise_attr_name(k);
let nv: Result<Set<Value>, _> =
let nv: Result<ValueSet, _> =
v.iter().map(|vr| qs.clone_value(audit, &nk, vr)).collect();
match nv {
Ok(nvi) => Ok((nk, nvi)),
@ -325,10 +326,10 @@ impl Entry<EntryInit, EntryNew> {
// str -> proto entry
let pe: ProtoEntry = serde_json::from_str(es).expect("Invalid Proto Entry");
// use a const map to convert str -> ava
let x: Map<AttrString, Set<Value>> = pe.attrs.into_iter()
let x: Map<AttrString, ValueSet> = pe.attrs.into_iter()
.map(|(k, vs)| {
let attr = AttrString::from(k.to_lowercase());
let vv: Set<Value> = match attr.as_str() {
let vv: ValueSet = match attr.as_str() {
"attributename" | "classname" | "domain" => {
vs.into_iter().map(|v| Value::new_iutf8(&v)).collect()
}
@ -500,8 +501,12 @@ impl Entry<EntryInit, EntryNew> {
}
/// Replace the existing content of an attribute set of this Entry, with a new set of Values.
pub fn set_ava(&mut self, attr: &str, values: Set<Value>) {
self.set_ava_int(attr, values)
// pub fn set_ava(&mut self, attr: &str, values: Set<Value>) {
pub fn set_ava<T>(&mut self, attr: &str, iter: T)
where
T: IntoIterator<Item = Value>,
{
self.set_ava_int(attr, iter)
}
}
@ -1290,12 +1295,12 @@ impl Entry<EntrySealed, EntryCommitted> {
pub fn from_dbentry(au: &mut AuditScope, db_e: DbEntry, id: u64) -> Result<Self, ()> {
// Convert attrs from db format to value
let r_attrs: Result<Map<AttrString, Set<Value>>, ()> = match db_e.ent {
let r_attrs: Result<Map<AttrString, ValueSet>, ()> = match db_e.ent {
DbEntryVers::V1(v1) => v1
.attrs
.into_iter()
.map(|(k, vs)| {
let vv: Result<Set<Value>, ()> =
let vv: Result<ValueSet, ()> =
vs.into_iter().map(Value::from_db_valuev1).collect();
match vv {
Ok(vv) => Ok((k, vv)),
@ -1377,14 +1382,14 @@ impl Entry<EntrySealed, EntryCommitted> {
/// Convert this recycled entry, into a tombstone ready for reaping.
pub fn to_tombstone(&self, cid: Cid) -> Entry<EntryInvalid, EntryCommitted> {
// Duplicate this to a tombstone entry
let class_ava = btreeset![Value::new_class("object"), Value::new_class("tombstone")];
let last_mod_ava = btreeset![Value::new_cid(cid.clone())];
let class_ava = valueset![Value::new_class("object"), Value::new_class("tombstone")];
let last_mod_ava = valueset![Value::new_cid(cid.clone())];
let mut attrs_new: Map<AttrString, Set<Value>> = Map::new();
let mut attrs_new: Map<AttrString, ValueSet> = Map::new();
attrs_new.insert(
AttrString::from("uuid"),
btreeset![Value::new_uuidr(&self.get_uuid())],
valueset![Value::new_uuidr(&self.get_uuid())],
);
attrs_new.insert(AttrString::from("class"), class_ava);
attrs_new.insert(AttrString::from("last_modified_cid"), last_mod_ava);
@ -1575,21 +1580,26 @@ impl<VALID, STATE> Entry<VALID, STATE> {
let v = self
.attrs
.entry(AttrString::from(attr))
.or_insert_with(Set::new);
.or_insert_with(ValueSet::new);
// Here we need to actually do a check/binary search ...
v.insert(value);
// Doesn't matter if it already exists, equality will replace.
}
/// Overwrite the current set of values for an attribute, with this new set.
pub fn set_ava_int(&mut self, attr: &str, values: Set<Value>) {
// pub fn set_ava_int(&mut self, attr: &str, values: Set<Value>) {
pub fn set_ava_int<T>(&mut self, attr: &str, iter: T)
where
T: IntoIterator<Item = Value>,
{
// Overwrite the existing value, build a tree from the list.
let values = iter.into_iter().collect();
let _ = self.attrs.insert(AttrString::from(attr), values);
}
/// Update the last_changed flag of this entry to the given change identifier.
fn set_last_changed(&mut self, cid: Cid) {
let cv = btreeset![Value::new_cid(cid)];
let cv = valueset![Value::new_cid(cid)];
let _ = self.attrs.insert(AttrString::from("last_modified_cid"), cv);
}
@ -1608,7 +1618,7 @@ impl<VALID, STATE> Entry<VALID, STATE> {
#[inline(always)]
/// Return a reference to the current set of values that are associated to this attribute.
pub fn get_ava_set(&self, attr: &str) -> Option<&Set<Value>> {
pub fn get_ava_set(&self, attr: &str) -> Option<&ValueSet> {
self.attrs.get(attr)
}
@ -1755,24 +1765,20 @@ impl<VALID, STATE> Entry<VALID, STATE> {
/// Assert if an attribute of this name is present, and one of it's values contains
/// the following substring, if possible to perform the substring comparison.
pub fn attribute_substring(&self, attr: &str, subvalue: &PartialValue) -> bool {
match self.attrs.get(attr) {
Some(v_list) => v_list
.iter()
.fold(false, |acc, v| if acc { acc } else { v.contains(subvalue) }),
None => false,
}
self.attrs
.get(attr)
.map(|vset| vset.substring(subvalue))
.unwrap_or(false)
}
#[inline(always)]
/// Assert if an attribute of this name is present, and one of it's values is less than
/// the following partial value
pub fn attribute_lessthan(&self, attr: &str, subvalue: &PartialValue) -> bool {
match self.attrs.get(attr) {
Some(v_list) => v_list
.iter()
.fold(false, |acc, v| if acc { acc } else { v.lessthan(subvalue) }),
None => false,
}
self.attrs
.get(attr)
.map(|vset| vset.lessthan(subvalue))
.unwrap_or(false)
}
// Since EntryValid/Invalid is just about class adherenece, not Value correctness, we
@ -1946,13 +1952,17 @@ where
}
/// Remove all values of this attribute from the entry, and return their content.
pub fn pop_ava(&mut self, attr: &str) -> Option<Set<Value>> {
pub fn pop_ava(&mut self, attr: &str) -> Option<ValueSet> {
self.attrs.remove(attr)
}
/// Replace the content of this attribute with a new value set.
pub fn set_ava(&mut self, attr: &str, values: Set<Value>) {
self.set_ava_int(attr, values)
// pub fn set_ava(&mut self, attr: &str, values: Set<Value>) {
pub fn set_ava<T>(&mut self, attr: &str, iter: T)
where
T: IntoIterator<Item = Value>,
{
self.set_ava_int(attr, iter)
}
/*
@ -1999,21 +2009,21 @@ impl<VALID, STATE> PartialEq for Entry<VALID, STATE> {
impl From<&SchemaAttribute> for Entry<EntryInit, EntryNew> {
fn from(s: &SchemaAttribute) -> Self {
// Convert an Attribute to an entry ... make it good!
let uuid_v = btreeset![Value::new_uuidr(&s.uuid)];
let uuid_v = valueset![Value::new_uuidr(&s.uuid)];
let name_v = btreeset![Value::new_iutf8(s.name.as_str())];
let desc_v = btreeset![Value::new_utf8(s.description.clone())];
let name_v = valueset![Value::new_iutf8(s.name.as_str())];
let desc_v = valueset![Value::new_utf8(s.description.clone())];
let multivalue_v = btreeset![Value::from(s.multivalue)];
let unique_v = btreeset![Value::from(s.unique)];
let multivalue_v = valueset![Value::from(s.multivalue)];
let unique_v = valueset![Value::from(s.unique)];
let index_v: Set<_> = s.index.iter().map(|i| Value::from(i.clone())).collect();
let index_v: ValueSet = s.index.iter().cloned().map(Value::from).collect();
let syntax_v = btreeset![Value::from(s.syntax.clone())];
let syntax_v = valueset![Value::from(s.syntax.clone())];
// Build the Map of the attributes relevant
// let mut attrs: Map<AttrString, Set<Value>> = Map::with_capacity(8);
let mut attrs: Map<AttrString, Set<Value>> = Map::new();
let mut attrs: Map<AttrString, ValueSet> = Map::new();
attrs.insert(AttrString::from("attributename"), name_v);
attrs.insert(AttrString::from("description"), desc_v);
attrs.insert(AttrString::from("uuid"), uuid_v);
@ -2023,7 +2033,7 @@ impl From<&SchemaAttribute> for Entry<EntryInit, EntryNew> {
attrs.insert(AttrString::from("syntax"), syntax_v);
attrs.insert(
AttrString::from("class"),
btreeset![
valueset![
Value::new_class("object"),
Value::new_class("system"),
Value::new_class("attributetype")
@ -2042,19 +2052,19 @@ impl From<&SchemaAttribute> for Entry<EntryInit, EntryNew> {
impl From<&SchemaClass> for Entry<EntryInit, EntryNew> {
fn from(s: &SchemaClass) -> Self {
let uuid_v = btreeset![Value::new_uuidr(&s.uuid)];
let uuid_v = valueset![Value::new_uuidr(&s.uuid)];
let name_v = btreeset![Value::new_iutf8(s.name.as_str())];
let desc_v = btreeset![Value::new_utf8(s.description.clone())];
let name_v = valueset![Value::new_iutf8(s.name.as_str())];
let desc_v = valueset![Value::new_utf8(s.description.clone())];
// let mut attrs: Map<AttrString, Set<Value>> = Map::with_capacity(8);
let mut attrs: Map<AttrString, Set<Value>> = Map::new();
let mut attrs: Map<AttrString, ValueSet> = Map::new();
attrs.insert(AttrString::from("classname"), name_v);
attrs.insert(AttrString::from("description"), desc_v);
attrs.insert(AttrString::from("uuid"), uuid_v);
attrs.insert(
AttrString::from("class"),
btreeset![
valueset![
Value::new_class("object"),
Value::new_class("system"),
Value::new_class("classtype")

View file

@ -47,6 +47,7 @@ mod interval;
pub(crate) mod ldap;
mod modify;
pub mod value;
pub mod valueset;
#[macro_use]
mod plugins;
mod access;

View file

@ -145,6 +145,7 @@ macro_rules! entry_str_to_account {
use crate::entry::{Entry, EntryInvalid, EntryNew};
use crate::idm::account::Account;
use crate::value::Value;
use std::iter::once;
let mut e: Entry<EntryInvalid, EntryNew> =
unsafe { Entry::unsafe_from_entry_str($entry_str).into_invalid_new() };
@ -153,7 +154,7 @@ macro_rules! entry_str_to_account {
.get_ava_single_str("name")
.map(|s| Value::new_spn_str(s, "example.com"))
.expect("Failed to munge spn from name!");
e.set_ava("spn", btreeset![spn]);
e.set_ava("spn", once(spn));
let e = unsafe { e.into_sealed_committed() };
@ -576,6 +577,30 @@ macro_rules! btreeset {
});
}
#[allow(unused_macros)]
#[macro_export]
macro_rules! valueset {
() => (
compile_error!("ValueSet needs at least 1 element")
);
($e:expr) => ({
use crate::valueset::ValueSet;
let mut x: ValueSet = ValueSet::new();
assert!(x.insert($e));
x
});
($e:expr,) => ({
valueset!($e)
});
($e:expr, $($item:expr),*) => ({
use crate::valueset::ValueSet;
let mut x: ValueSet = ValueSet::new();
assert!(x.insert($e));
$(assert!(x.insert($item));)*
x
});
}
#[allow(unused_macros)]
#[macro_export]
macro_rules! entry_init {

View file

@ -12,14 +12,13 @@
use crate::entry::{Entry, EntryCommitted, EntryInvalid, EntrySealed};
use crate::event::{CreateEvent, DeleteEvent, ModifyEvent};
use crate::prelude::*;
// use crate::modify::{Modify, ModifyList};
use crate::plugins::Plugin;
use crate::prelude::*;
use crate::value::{PartialValue, Value};
use crate::valueset::ValueSet;
use kanidm_proto::v1::{ConsistencyError, OperationError};
use hashbrown::HashMap;
use std::collections::BTreeSet;
use uuid::Uuid;
lazy_static! {
@ -351,7 +350,7 @@ impl Plugin for MemberOf {
};
// for all direct -> add uuid to map
let d_groups_set: BTreeSet<_> = direct_memberof
let d_groups_set: ValueSet = direct_memberof
.iter()
.map(|e| Value::new_refer(*e.get_uuid()))
.collect();

View file

@ -19,11 +19,12 @@
use crate::audit::AuditScope;
use crate::be::IdxKey;
use crate::prelude::*;
use crate::valueset::ValueSet;
use kanidm_proto::v1::{ConsistencyError, OperationError, SchemaError};
use hashbrown::{HashMap, HashSet};
use std::borrow::Borrow;
use std::collections::BTreeSet;
// use std::collections::BTreeSet;
use uuid::Uuid;
// use concread::cowcell::asynch::*;
@ -222,7 +223,7 @@ impl SchemaAttribute {
}
}
pub fn validate_ava(&self, a: &str, ava: &BTreeSet<Value>) -> Result<(), SchemaError> {
pub fn validate_ava(&self, a: &str, ava: &ValueSet) -> Result<(), SchemaError> {
// ltrace!("Checking for valid {:?} -> {:?}", self.name, ava);
// Check multivalue
if !self.multivalue && ava.len() > 1 {
@ -1737,12 +1738,12 @@ mod tests {
};
let r1 =
single_value_string.validate_ava("single_value", &btreeset![Value::new_iutf8("test")]);
single_value_string.validate_ava("single_value", &valueset![Value::new_iutf8("test")]);
assert_eq!(r1, Ok(()));
let r2 = single_value_string.validate_ava(
"single_value",
&btreeset![Value::new_iutf8("test1"), Value::new_iutf8("test2")],
&valueset![Value::new_iutf8("test1"), Value::new_iutf8("test2")],
);
assert_eq!(
r2,
@ -1767,7 +1768,7 @@ mod tests {
let r5 = multi_value_string.validate_ava(
"mv_string",
&btreeset![Value::new_utf8s("test1"), Value::new_utf8s("test2")],
&valueset![Value::new_utf8s("test1"), Value::new_utf8s("test2")],
);
assert_eq!(r5, Ok(()));
@ -1785,7 +1786,7 @@ mod tests {
let r3 = multi_value_boolean.validate_ava(
"mv_bool",
&btreeset![
&valueset![
Value::new_bool(true),
Value::new_iutf8("test1"),
Value::new_iutf8("test2")
@ -1798,7 +1799,7 @@ mod tests {
let r4 = multi_value_boolean.validate_ava(
"mv_bool",
&btreeset![Value::new_bool(true), Value::new_bool(false)],
&valueset![Value::new_bool(true), Value::new_bool(false)],
);
assert_eq!(r4, Ok(()));
@ -1817,12 +1818,12 @@ mod tests {
let r6 = single_value_syntax.validate_ava(
"sv_syntax",
&btreeset![Value::new_syntaxs("UTF8STRING").unwrap()],
&valueset![Value::new_syntaxs("UTF8STRING").unwrap()],
);
assert_eq!(r6, Ok(()));
let r7 = single_value_syntax
.validate_ava("sv_syntax", &btreeset![Value::new_utf8s("thaeountaheu")]);
.validate_ava("sv_syntax", &valueset![Value::new_utf8s("thaeountaheu")]);
assert_eq!(
r7,
Err(SchemaError::InvalidAttributeSyntax("sv_syntax".to_string()))
@ -1842,12 +1843,12 @@ mod tests {
//
let r8 = single_value_index.validate_ava(
"sv_index",
&btreeset![Value::new_indexs("EQUALITY").unwrap()],
&valueset![Value::new_indexs("EQUALITY").unwrap()],
);
assert_eq!(r8, Ok(()));
let r9 = single_value_index
.validate_ava("sv_index", &btreeset![Value::new_utf8s("thaeountaheu")]);
.validate_ava("sv_index", &valueset![Value::new_utf8s("thaeountaheu")]);
assert_eq!(
r9,
Err(SchemaError::InvalidAttributeSyntax("sv_index".to_string()))

View file

@ -558,7 +558,7 @@ impl PartialValue {
}
}
pub fn contains(&self, s: &PartialValue) -> bool {
pub fn substring(&self, s: &PartialValue) -> bool {
match (self, s) {
(PartialValue::Utf8(s1), PartialValue::Utf8(s2)) => s1.contains(s2),
(PartialValue::Iutf8(s1), PartialValue::Iutf8(s2)) => s1.contains(s2),
@ -620,10 +620,10 @@ impl PartialValue {
/// or modification operation where you are applying a set of complete values into an entry.
#[derive(Clone, Debug)]
pub struct Value {
pv: PartialValue,
pub(crate) pv: PartialValue,
// Later we'll add extra data fields for different v types. They'll have to switch on
// pv somehow, so probably need optional or union?
data: Option<Box<DataValue>>,
pub(crate) data: Option<Box<DataValue>>,
}
// TODO: Impl display
@ -1072,14 +1072,14 @@ impl Value {
self.pv.is_url()
}
pub fn contains(&self, s: &PartialValue) -> bool {
self.pv.contains(s)
}
pub fn lessthan(&self, s: &PartialValue) -> bool {
self.pv.lessthan(s)
}
pub fn substring(&self, s: &PartialValue) -> bool {
self.pv.substring(s)
}
// Converters between DBRepr -> MemRepr. It's likely many of these
// will be just wrappers to our from str types.

210
kanidmd/src/lib/valueset.rs Normal file
View file

@ -0,0 +1,210 @@
use crate::prelude::*;
// use hashbrown::HashSet;
use std::borrow::Borrow;
use std::collections::BTreeSet;
use std::iter::FromIterator;
pub struct ValueSet {
inner: BTreeSet<Value>,
}
impl Default for ValueSet {
fn default() -> Self {
ValueSet {
inner: BTreeSet::new(),
}
}
}
impl ValueSet {
pub fn new() -> Self {
Self::default()
}
// insert
pub fn insert(&mut self, value: Value) -> bool {
// Return true if the element is new.
self.inner.insert(value)
}
// set values
pub fn set(&mut self, iter: impl Iterator<Item = Value>) {
self.inner.clear();
self.inner.extend(iter);
}
pub fn get<Q>(&self, value: &Q) -> Option<&Value>
where
Value: Borrow<Q> + Ord,
Q: Ord + ?Sized,
{
self.inner.get(value)
}
// delete a value
pub fn remove<Q>(&mut self, value: &Q) -> bool
where
Value: Borrow<Q> + Ord,
Q: Ord + ?Sized,
{
self.inner.remove(value)
}
pub fn contains<Q>(&self, value: &Q) -> bool
where
Value: Borrow<Q> + Ord,
Q: Ord + ?Sized,
{
self.inner.contains(value)
}
pub fn substring(&self, value: &PartialValue) -> bool {
self.inner.iter().any(|v| v.substring(value))
}
pub fn lessthan(&self, value: &PartialValue) -> bool {
self.inner.iter().any(|v| v.lessthan(value))
}
pub fn len(&self) -> usize {
self.inner.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
// We'll need to be able to do partialeq/intersect etc later.
pub fn iter(&self) -> Iter {
(&self).into_iter()
}
pub fn difference<'a>(&'a self, other: &'a ValueSet) -> Difference<'a> {
Difference {
iter: self.inner.difference(&other.inner),
}
}
pub fn symmetric_difference<'a>(&'a self, other: &'a ValueSet) -> SymmetricDifference<'a> {
SymmetricDifference {
iter: self.inner.symmetric_difference(&other.inner),
}
}
}
impl PartialEq for ValueSet {
fn eq(&self, other: &Self) -> bool {
self.inner.eq(&other.inner)
}
}
impl FromIterator<Value> for ValueSet {
fn from_iter<T>(iter: T) -> Self
where
T: IntoIterator<Item = Value>,
{
ValueSet {
inner: BTreeSet::from_iter(iter),
}
}
}
impl Clone for ValueSet {
fn clone(&self) -> Self {
ValueSet {
inner: self.inner.clone(),
}
}
}
pub struct Iter<'a> {
iter: std::collections::btree_set::Iter<'a, Value>,
}
impl<'a> Iterator for Iter<'a> {
type Item = &'a Value;
fn next(&mut self) -> Option<&'a Value> {
self.iter.next()
}
}
impl<'a> IntoIterator for &'a ValueSet {
type Item = &'a Value;
type IntoIter = Iter<'a>;
fn into_iter(self) -> Self::IntoIter {
Iter {
iter: (&self.inner).into_iter(),
}
}
}
pub struct Difference<'a> {
iter: std::collections::btree_set::Difference<'a, Value>,
}
impl<'a> Iterator for Difference<'a> {
type Item = &'a Value;
fn next(&mut self) -> Option<&'a Value> {
self.iter.next()
}
}
pub struct SymmetricDifference<'a> {
iter: std::collections::btree_set::SymmetricDifference<'a, Value>,
}
impl<'a> Iterator for SymmetricDifference<'a> {
type Item = &'a Value;
fn next(&mut self) -> Option<&'a Value> {
self.iter.next()
}
}
pub struct IntoIter {
iter: std::collections::btree_set::IntoIter<Value>,
}
impl Iterator for IntoIter {
type Item = Value;
fn next(&mut self) -> Option<Value> {
self.iter.next()
}
}
impl IntoIterator for ValueSet {
type Item = Value;
type IntoIter = IntoIter;
fn into_iter(self) -> Self::IntoIter {
IntoIter {
iter: self.inner.into_iter(),
}
}
}
impl std::fmt::Debug for ValueSet {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ValueSet")
.field("inner", &self.inner)
.finish()
}
}
#[cfg(test)]
mod tests {
use crate::value::Value;
use crate::valueset::ValueSet;
#[test]
fn test_valueset_basic() {
let mut vs = ValueSet::new();
assert!(vs.insert(Value::new_uint32(0)));
assert!(!vs.insert(Value::new_uint32(0)));
}
}