kanidm/server/lib/src/be/mod.rs

//! The backend. This contains the "low level" storage and query code, which is
//! implemented as a json-like kv document database. This has no rules about content
//! of the server, which are all enforced at higher levels. The role of the backend
//! is to persist content safely to disk, load that content, and execute queries
//! utilising indexes in the most effective way possible.

use std::fs;
use std::ops::DerefMut;
use std::sync::Arc;
use std::time::Duration;

use concread::cowcell::*;
use hashbrown::{HashMap as Map, HashSet};
use idlset::v2::IDLBitRange;
use idlset::AndNot;
use kanidm_proto::v1::{ConsistencyError, OperationError};
use smartstring::alias::String as AttrString;
use tracing::{trace, trace_span};
use uuid::Uuid;

use crate::be::dbentry::{DbBackup, DbEntry};
use crate::entry::{Entry, EntryCommitted, EntryNew, EntrySealed};
use crate::filter::{Filter, FilterPlan, FilterResolved, FilterValidResolved};
use crate::prelude::*;
use crate::repl::cid::Cid;
use crate::repl::ruv::{
    ReplicationUpdateVector, ReplicationUpdateVectorReadTransaction,
    ReplicationUpdateVectorTransaction, ReplicationUpdateVectorWriteTransaction,
};
use crate::value::{IndexType, Value};

pub mod dbentry;
pub mod dbvalue;
mod idl_arc_sqlite;
mod idl_sqlite;
pub(crate) mod idxkey;

pub(crate) use self::idxkey::{IdxKey, IdxKeyRef, IdxKeyToRef, IdxSlope};
use crate::be::idl_arc_sqlite::{
    IdlArcSqlite, IdlArcSqliteReadTransaction, IdlArcSqliteTransaction,
    IdlArcSqliteWriteTransaction,
};
// Re-export this
pub use crate::be::idl_sqlite::FsType;

// Currently disabled due to improvements in idlset for intersection handling.
const FILTER_SEARCH_TEST_THRESHOLD: usize = 0;
const FILTER_EXISTS_TEST_THRESHOLD: usize = 0;

#[derive(Debug, Clone)]
/// Limits on the resources a single event can consume. These are defined per-event
/// as they are derived from the userAuthToken based on that individual session
pub struct Limits {
    pub unindexed_allow: bool,
    pub search_max_results: usize,
    pub search_max_filter_test: usize,
    pub filter_max_elements: usize,
}

impl Default for Limits {
    fn default() -> Self {
        Limits {
            unindexed_allow: false,
            search_max_results: 256,
            search_max_filter_test: 512,
            filter_max_elements: 32,
        }
    }
}

impl Limits {
    pub fn unlimited() -> Self {
        Limits {
            unindexed_allow: true,
            search_max_results: usize::MAX,
            search_max_filter_test: usize::MAX,
            filter_max_elements: usize::MAX,
        }
    }
}

#[derive(Debug, Clone)]
pub enum IdList {
    AllIds,
    PartialThreshold(IDLBitRange),
    Partial(IDLBitRange),
    Indexed(IDLBitRange),
}

#[derive(Debug)]
pub struct IdRawEntry {
    id: u64,
    data: Vec<u8>,
}

#[derive(Debug, Clone)]
pub struct IdxMeta {
    pub idxkeys: Map<IdxKey, IdxSlope>,
}

impl IdxMeta {
    pub fn new(idxkeys: Map<IdxKey, IdxSlope>) -> Self {
        IdxMeta { idxkeys }
    }
}

#[derive(Clone)]
pub struct BackendConfig {
    path: String,
    pool_size: u32,
    db_name: &'static str,
    fstype: FsType,
    // Cachesizes?
    arcsize: Option<usize>,
}

impl BackendConfig {
    pub fn new(path: &str, pool_size: u32, fstype: FsType, arcsize: Option<usize>) -> Self {
        BackendConfig {
            pool_size,
            path: path.to_string(),
            db_name: "main",
            fstype,
            arcsize,
        }
    }

    pub(crate) fn new_test(db_name: &'static str) -> Self {
        BackendConfig {
            pool_size: 1,
            path: "".to_string(),
            db_name,
            fstype: FsType::Generic,
            arcsize: Some(1024),
        }
    }
}

#[derive(Clone)]
pub struct Backend {
    /// This is the actual datastorage layer.
    idlayer: Arc<IdlArcSqlite>,
    /// This is a copy-on-write cache of the index metadata that has been
    /// extracted from attributes set, in the correct format for the backend
    /// to consume. We use it to extract indexes from entries during write paths
    /// and to allow the front end to know what indexes exist during a read.
    idxmeta: Arc<CowCell<IdxMeta>>,
    /// The current state of the replication update vector. This is effectively a
    /// time series index of the full list of all changelog entries and what entries
    /// that are part of that change.
    ruv: Arc<ReplicationUpdateVector>,
    cfg: BackendConfig,
}

pub struct BackendReadTransaction<'a> {
    idlayer: IdlArcSqliteReadTransaction<'a>,
    idxmeta: CowCellReadTxn<IdxMeta>,
    ruv: ReplicationUpdateVectorReadTransaction<'a>,
}

unsafe impl<'a> Sync for BackendReadTransaction<'a> {}

unsafe impl<'a> Send for BackendReadTransaction<'a> {}

pub struct BackendWriteTransaction<'a> {
    idlayer: IdlArcSqliteWriteTransaction<'a>,
    idxmeta_wr: CowCellWriteTxn<'a, IdxMeta>,
    ruv: ReplicationUpdateVectorWriteTransaction<'a>,
}

impl IdRawEntry {
    fn into_dbentry(self) -> Result<(u64, DbEntry), OperationError> {
        serde_json::from_slice(self.data.as_slice())
            .map_err(|e| {
                admin_error!(?e, "Serde JSON Error");
                OperationError::SerdeJsonError
            })
            .map(|dbe| (self.id, dbe))
    }

    fn into_entry(self) -> Result<Entry<EntrySealed, EntryCommitted>, OperationError> {
        let db_e = serde_json::from_slice(self.data.as_slice()).map_err(|e| {
            admin_error!(?e, id = %self.id, "Serde JSON Error");
            let raw_str = String::from_utf8_lossy(self.data.as_slice());
            debug!(raw = %raw_str);
            OperationError::SerdeJsonError
        })?;
        // let id = u64::try_from(self.id).map_err(|_| OperationError::InvalidEntryId)?;
        Entry::from_dbentry(db_e, self.id).ok_or(OperationError::CorruptedEntry(self.id))
    }
}

pub trait BackendTransaction {
    type IdlLayerType: IdlArcSqliteTransaction;
    fn get_idlayer(&mut self) -> &mut Self::IdlLayerType;

    type RuvType: ReplicationUpdateVectorTransaction;
    fn get_ruv(&mut self) -> &mut Self::RuvType;

    fn get_idxmeta_ref(&self) -> &IdxMeta;

    /// Recursively apply a filter, transforming into IdList's on the way. This builds a query
    /// execution log, so that it can be examined how an operation proceeded.
    #[allow(clippy::cognitive_complexity)]
    #[instrument(level = "debug", name = "be::filter2idl", skip_all)]
    fn filter2idl(
        &mut self,
        filt: &FilterResolved,
        thres: usize,
    ) -> Result<(IdList, FilterPlan), OperationError> {
        Ok(match filt {
            FilterResolved::Eq(attr, value, idx) => {
                if idx.is_some() {
                    // Get the idx_key
                    let idx_key = value.get_idx_eq_key();
                    // Get the idl for this
                    match self
                        .get_idlayer()
                        .get_idl(attr, IndexType::Equality, &idx_key)?
                    {
                        Some(idl) => (
                            IdList::Indexed(idl),
                            FilterPlan::EqIndexed(attr.clone(), idx_key),
                        ),
                        None => (IdList::AllIds, FilterPlan::EqCorrupt(attr.clone())),
                    }
                } else {
                    // Schema believes this is not indexed
                    (IdList::AllIds, FilterPlan::EqUnindexed(attr.clone()))
                }
            }
            FilterResolved::Sub(attr, subvalue, idx) => {
                if idx.is_some() {
                    // Get the idx_key
                    let idx_key = subvalue.get_idx_sub_key();
                    // Get the idl for this
                    match self
                        .get_idlayer()
                        .get_idl(attr, IndexType::SubString, &idx_key)?
                    {
                        Some(idl) => (
                            IdList::Indexed(idl),
                            FilterPlan::SubIndexed(attr.clone(), idx_key),
                        ),
                        None => (IdList::AllIds, FilterPlan::SubCorrupt(attr.clone())),
                    }
                } else {
                    // Schema believes this is not indexed
                    (IdList::AllIds, FilterPlan::SubUnindexed(attr.clone()))
                }
            }
            FilterResolved::Pres(attr, idx) => {
                if idx.is_some() {
                    // Get the idl for this
                    match self.get_idlayer().get_idl(attr, IndexType::Presence, "_")? {
                        Some(idl) => (IdList::Indexed(idl), FilterPlan::PresIndexed(attr.clone())),
                        None => (IdList::AllIds, FilterPlan::PresCorrupt(attr.clone())),
                    }
                } else {
                    // Schema believes this is not indexed
                    (IdList::AllIds, FilterPlan::PresUnindexed(attr.clone()))
                }
            }
            FilterResolved::LessThan(attr, _subvalue, _idx) => {
                // We have no process for indexing this right now.
                (IdList::AllIds, FilterPlan::LessThanUnindexed(attr.clone()))
            }
            FilterResolved::Or(l, _) => {
                // Importantly if this has no inner elements, this returns
                // an empty list.
                let mut plan = Vec::new();
                let mut result = IDLBitRange::new();
                let mut partial = false;
                let mut threshold = false;
                // For each filter in l
                for f in l.iter() {
                    // get their idls
                    match self.filter2idl(f, thres)? {
                        (IdList::Indexed(idl), fp) => {
                            plan.push(fp);
                            // now union them (if possible)
                            result = result | idl;
                        }
                        (IdList::Partial(idl), fp) => {
                            plan.push(fp);
                            // now union them (if possible)
                            result = result | idl;
                            partial = true;
                        }
                        (IdList::PartialThreshold(idl), fp) => {
                            plan.push(fp);
                            // now union them (if possible)
                            result = result | idl;
                            partial = true;
                            threshold = true;
                        }
                        (IdList::AllIds, fp) => {
                            plan.push(fp);
                            // If we find anything unindexed, the whole term is unindexed.
                            filter_trace!("Term {:?} is AllIds, shortcut return", f);
                            let setplan = FilterPlan::OrUnindexed(plan);
                            return Ok((IdList::AllIds, setplan));
                        }
                    }
                } // end or.iter()
                  // If we got here, every term must have been indexed or partial indexed.
                if partial {
                    if threshold {
                        let setplan = FilterPlan::OrPartialThreshold(plan);
                        (IdList::PartialThreshold(result), setplan)
                    } else {
                        let setplan = FilterPlan::OrPartial(plan);
                        (IdList::Partial(result), setplan)
                    }
                } else {
                    let setplan = FilterPlan::OrIndexed(plan);
                    (IdList::Indexed(result), setplan)
                }
            }
            FilterResolved::And(l, _) => {
                // This algorithm is a little annoying. I couldn't get it to work with iter and
                // folds due to the logic needed ...

                // First, setup the two filter lists. We always apply AndNot after positive
                // and terms.
                let (f_andnot, f_rem): (Vec<_>, Vec<_>) = l.iter().partition(|f| f.is_andnot());

                // We make this an iter, so everything comes off in order. if we used pop it means we
                // pull from the tail, which is the WORST item to start with!
                let mut f_rem_iter = f_rem.iter();

                // Setup the initial result.
                let (mut cand_idl, fp) = match f_rem_iter.next() {
                    Some(f) => self.filter2idl(f, thres)?,
                    None => {
                        filter_warn!(
                            "And filter was empty, or contains only AndNot, can not evaluate."
                        );
                        return Ok((IdList::Indexed(IDLBitRange::new()), FilterPlan::Invalid));
                    }
                };

                // Setup the counter of terms we have left to evaluate.
                // This is used so that we shortcut return ONLY when we really do have
                // more terms remaining.
                let mut f_rem_count = f_rem.len() + f_andnot.len() - 1;

                // Setup the query plan tracker
                let mut plan = vec![fp];

                match &cand_idl {
                    IdList::Indexed(idl) | IdList::Partial(idl) | IdList::PartialThreshold(idl) => {
                        // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                        // But we only do this when there are actually multiple elements in the and,
                        // because an and with 1 element now is FULLY resolved.
                        if idl.below_threshold(thres) && f_rem_count > 0 {
                            let setplan = FilterPlan::AndPartialThreshold(plan);
                            return Ok((IdList::PartialThreshold(idl.clone()), setplan));
                        } else if idl.is_empty() {
                            // Regardless of the input state, if it's empty, this can never
                            // be satisfied, so return we are indexed and complete.
                            let setplan = FilterPlan::AndEmptyCand(plan);
                            return Ok((IdList::Indexed(IDLBitRange::new()), setplan));
                        }
                    }
                    IdList::AllIds => {}
                }

                // Now, for all remaining,
                for f in f_rem_iter {
                    f_rem_count -= 1;
                    let (inter, fp) = self.filter2idl(f, thres)?;
                    plan.push(fp);
                    cand_idl = match (cand_idl, inter) {
                        (IdList::Indexed(ia), IdList::Indexed(ib)) => {
                            let r = ia & ib;
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else if r.is_empty() {
                                // Regardless of the input state, if it's empty, this can never
                                // be satisfied, so return we are indexed and complete.
                                let setplan = FilterPlan::AndEmptyCand(plan);
                                return Ok((IdList::Indexed(IDLBitRange::new()), setplan));
                            } else {
                                IdList::Indexed(r)
                            }
                        }
                        (IdList::Indexed(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::Indexed(ib))
                        | (IdList::Partial(ia), IdList::Partial(ib)) => {
                            let r = ia & ib;
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::Partial(r)
                            }
                        }
                        (IdList::Indexed(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Indexed(ib))
                        | (IdList::PartialThreshold(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::PartialThreshold(ib)) => {
                            let r = ia & ib;
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::PartialThreshold(r)
                            }
                        }
                        (IdList::Indexed(i), IdList::AllIds)
                        | (IdList::AllIds, IdList::Indexed(i))
                        | (IdList::Partial(i), IdList::AllIds)
                        | (IdList::AllIds, IdList::Partial(i)) => IdList::Partial(i),
                        (IdList::PartialThreshold(i), IdList::AllIds)
                        | (IdList::AllIds, IdList::PartialThreshold(i)) => {
                            IdList::PartialThreshold(i)
                        }
                        (IdList::AllIds, IdList::AllIds) => IdList::AllIds,
                    };
                }

                // debug!("partial cand set ==> {:?}", cand_idl);

                for f in f_andnot.iter() {
                    f_rem_count -= 1;
                    let f_in = match f {
                        FilterResolved::AndNot(f_in, _) => f_in,
                        _ => {
                            filter_error!(
                                "Invalid server state, a cand filter leaked to andnot set!"
                            );
                            return Err(OperationError::InvalidState);
                        }
                    };
                    let (inter, fp) = self.filter2idl(f_in, thres)?;
                    // It's an and not, so we need to wrap the plan accordingly.
                    plan.push(FilterPlan::AndNot(Box::new(fp)));
                    cand_idl = match (cand_idl, inter) {
                        (IdList::Indexed(ia), IdList::Indexed(ib)) => {
                            let r = ia.andnot(ib);
                            /*
                            // Don't trigger threshold on and nots if fully indexed.
                            if r.below_threshold(thres) {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                return Ok(IdList::PartialThreshold(r));
                            } else {
                                IdList::Indexed(r)
                            }
                            */
                            IdList::Indexed(r)
                        }
                        (IdList::Indexed(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::Indexed(ib))
                        | (IdList::Partial(ia), IdList::Partial(ib)) => {
                            let r = ia.andnot(ib);
                            // DO trigger threshold on partials, because we have to apply the filter
                            // test anyway, so we may as well shortcut at this point.
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::Partial(r)
                            }
                        }
                        (IdList::Indexed(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Indexed(ib))
                        | (IdList::PartialThreshold(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::PartialThreshold(ib)) => {
                            let r = ia.andnot(ib);
                            // DO trigger threshold on partials, because we have to apply the filter
                            // test anyway, so we may as well shortcut at this point.
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::PartialThreshold(r)
                            }
                        }

                        (IdList::Indexed(_), IdList::AllIds)
                        | (IdList::AllIds, IdList::Indexed(_))
                        | (IdList::Partial(_), IdList::AllIds)
                        | (IdList::AllIds, IdList::Partial(_))
                        | (IdList::PartialThreshold(_), IdList::AllIds)
                        | (IdList::AllIds, IdList::PartialThreshold(_)) => {
                            // We could actually generate allids here
                            // and then try to reduce the and-not set, but
                            // for now we just return all ids.
                            IdList::AllIds
                        }
                        (IdList::AllIds, IdList::AllIds) => IdList::AllIds,
                    };
                }

                // What state is the final cand idl in?
                let setplan = match cand_idl {
                    IdList::Indexed(_) => FilterPlan::AndIndexed(plan),
                    IdList::Partial(_) | IdList::PartialThreshold(_) => {
                        FilterPlan::AndPartial(plan)
                    }
                    IdList::AllIds => FilterPlan::AndUnindexed(plan),
                };

                // Finally, return the result.
                // debug!("final cand set ==> {:?}", cand_idl);
                (cand_idl, setplan)
            } // end and
            FilterResolved::Inclusion(l, _) => {
                // For inclusion to be valid, every term must have *at least* one element present.
                // This really relies on indexing, and so it's internal only - generally only
                // for fully indexed existence queries, such as from refint.

                // This has a lot in common with an And and Or but not really quite either.
                let mut plan = Vec::new();
                let mut result = IDLBitRange::new();
                // For each filter in l
                for f in l.iter() {
                    // get their idls
                    match self.filter2idl(f, thres)? {
                        (IdList::Indexed(idl), fp) => {
                            plan.push(fp);
                            if idl.is_empty() {
                                // It's empty, so something is missing. Bail fast.
                                filter_trace!("Inclusion is unable to proceed - an empty (missing) item was found!");
                                let setplan = FilterPlan::InclusionIndexed(plan);
                                return Ok((IdList::Indexed(IDLBitRange::new()), setplan));
                            } else {
                                result = result | idl;
                            }
                        }
                        (_, fp) => {
                            plan.push(fp);
                            filter_error!(
                                "Inclusion is unable to proceed - all terms must be fully indexed!"
                            );
                            let setplan = FilterPlan::InclusionInvalid(plan);
                            return Ok((IdList::Partial(IDLBitRange::new()), setplan));
                        }
                    }
                } // end or.iter()
                  // If we got here, every term must have been indexed
                let setplan = FilterPlan::InclusionIndexed(plan);
                (IdList::Indexed(result), setplan)
            }
            // So why does this return empty? Normally we actually process an AndNot in the context
            // of an "AND" query, but if it's used anywhere else IE the root filter, then there is
            // no other set to exclude - therefore it's empty set. Additionally, even in an OR query
            // the AndNot will be skipped as an empty set for the same reason.
            FilterResolved::AndNot(_f, _) => {
                // get the idl for f
                // now do andnot?
                filter_error!("Requested a top level or isolated AndNot, returning empty");
                (IdList::Indexed(IDLBitRange::new()), FilterPlan::Invalid)
            }
        })
    }

    #[instrument(level = "debug", name = "be::search", skip_all)]
    fn search(
        &mut self,
        erl: &Limits,
        filt: &Filter<FilterValidResolved>,
    ) -> Result<Vec<Arc<EntrySealedCommitted>>, OperationError> {
        let _entered = trace_span!("be::search").entered();
        // Unlike DS, even if we don't get the index back, we can just pass
        // to the in-memory filter test and be done.

        debug!(filter_optimised = ?filt);

        let (idl, fplan) = trace_span!("be::search -> filter2idl")
            .in_scope(|| self.filter2idl(filt.to_inner(), FILTER_SEARCH_TEST_THRESHOLD))?;

        debug!(filter_executed_plan = ?fplan);

        match &idl {
            IdList::AllIds => {
                if !erl.unindexed_allow {
                    admin_error!(
                        "filter (search) is fully unindexed, and not allowed by resource limits"
                    );
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::Partial(idl_br) => {
                // if idl_br.len() > erl.search_max_filter_test {
                if !idl_br.below_threshold(erl.search_max_filter_test) {
                    admin_error!("filter (search) is partial indexed and greater than search_max_filter_test allowed by resource limits");
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::PartialThreshold(_) => {
                // Since we opted for this, this is not the fault
                // of the user and we should not penalise them by limiting on partial.
            }
            IdList::Indexed(idl_br) => {
                // We know this is resolved here, so we can attempt the limit
                // check. This has to fold the whole index, but you know, class=pres is
                // indexed ...
                // if idl_br.len() > erl.search_max_results {
                if !idl_br.below_threshold(erl.search_max_results) {
                    admin_error!("filter (search) is indexed and greater than search_max_results allowed by resource limits");
                    return Err(OperationError::ResourceLimit);
                }
            }
        };

        let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
            admin_error!(?e, "get_identry failed");
            e
        })?;

        let entries_filtered = match idl {
            IdList::AllIds => trace_span!("be::search<entry::ftest::allids>").in_scope(|| {
                entries
                    .into_iter()
                    .filter(|e| e.entry_match_no_index(filt))
                    .collect()
            }),
            IdList::Partial(_) => trace_span!("be::search<entry::ftest::partial>").in_scope(|| {
                entries
                    .into_iter()
                    .filter(|e| e.entry_match_no_index(filt))
                    .collect()
            }),
            IdList::PartialThreshold(_) => trace_span!("be::search<entry::ftest::thresh>")
                .in_scope(|| {
                    entries
                        .into_iter()
                        .filter(|e| e.entry_match_no_index(filt))
                        .collect()
                }),
            // Since the index fully resolved, we can shortcut the filter test step here!
            IdList::Indexed(_) => {
                filter_trace!("filter (search) was fully indexed 👏");
                entries
            }
        };

        // If the idl was not indexed, apply the resource limit now. Avoid the needless match since the
        // if statement is quick.
        if entries_filtered.len() > erl.search_max_results {
            admin_error!("filter (search) is resolved and greater than search_max_results allowed by resource limits");
            return Err(OperationError::ResourceLimit);
        }

        Ok(entries_filtered)
    }

    /// Given a filter, assert some condition exists.
    /// Basically, this is a specialised case of search, where we don't need to
    /// load any candidates if they match. This is heavily used in uuid
    /// refint and attr uniqueness.
    #[instrument(level = "debug", name = "be::exists", skip_all)]
    fn exists(
        &mut self,
        erl: &Limits,
        filt: &Filter<FilterValidResolved>,
    ) -> Result<bool, OperationError> {
        debug!(filter_optimised = ?filt);

        // Using the indexes, resolve the IdList here, or AllIds.
        // Also get if the filter was 100% resolved or not.
        let (idl, fplan) = self.filter2idl(filt.to_inner(), FILTER_EXISTS_TEST_THRESHOLD)?;

        debug!(filter_executed_plan = ?fplan);

        // Apply limits to the IdList.
        match &idl {
            IdList::AllIds => {
                if !erl.unindexed_allow {
                    admin_error!(
                        "filter (exists) is fully unindexed, and not allowed by resource limits"
                    );
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::Partial(idl_br) => {
                if !idl_br.below_threshold(erl.search_max_filter_test) {
                    admin_error!("filter (exists) is partial indexed and greater than search_max_filter_test allowed by resource limits");
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::PartialThreshold(_) => {
                // Since we opted for this, this is not the fault
                // of the user and we should not penalise them.
            }
            IdList::Indexed(_) => {}
        }

        // Now, check the idl -- if it's fully resolved, we can skip this because the query
        // was fully indexed.
        match &idl {
            IdList::Indexed(idl) => Ok(!idl.is_empty()),
            _ => {
                let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
                    admin_error!(?e, "get_identry failed");
                    e
                })?;

                // if not 100% resolved query, apply the filter test.
                let entries_filtered: Vec<_> =
                    trace_span!("be::exists<entry::ftest>").in_scope(|| {
                        entries
                            .into_iter()
                            .filter(|e| e.entry_match_no_index(filt))
                            .collect()
                    });

                Ok(!entries_filtered.is_empty())
            }
        } // end match idl
    }

    fn verify(&mut self) -> Vec<Result<(), ConsistencyError>> {
        self.get_idlayer().verify()
    }

    fn verify_entry_index(
        &mut self,
        e: &Entry<EntrySealed, EntryCommitted>,
    ) -> Result<(), ConsistencyError> {
        // First, check our references in name2uuid, uuid2spn and uuid2rdn
        if e.mask_recycled_ts().is_some() {
            let e_uuid = e.get_uuid();
            // We only check these on live entries.
            let (n2u_add, n2u_rem) = Entry::idx_name2uuid_diff(None, Some(e));

            let n2u_set = match (n2u_add, n2u_rem) {
                (Some(set), None) => set,
                (_, _) => {
                    admin_error!("Invalid idx_name2uuid_diff state");
                    return Err(ConsistencyError::BackendIndexSync);
                }
            };

            // If the set.len > 1, check each item.
            n2u_set
                .iter()
                .try_for_each(|name| match self.get_idlayer().name2uuid(name) {
                    Ok(Some(idx_uuid)) => {
                        if idx_uuid == e_uuid {
                            Ok(())
                        } else {
                            admin_error!("Invalid name2uuid state -> incorrect uuid association");
                            Err(ConsistencyError::BackendIndexSync)
                        }
                    }
                    r => {
                        admin_error!(state = ?r, "Invalid name2uuid state");
                        Err(ConsistencyError::BackendIndexSync)
                    }
                })?;

            let spn = e.get_uuid2spn();
            match self.get_idlayer().uuid2spn(e_uuid) {
                Ok(Some(idx_spn)) => {
                    if spn != idx_spn {
                        admin_error!("Invalid uuid2spn state -> incorrect idx spn value");
                        return Err(ConsistencyError::BackendIndexSync);
                    }
                }
                r => {
                    admin_error!(state = ?r, "Invalid uuid2spn state");
                    return Err(ConsistencyError::BackendIndexSync);
                }
            };

            let rdn = e.get_uuid2rdn();
            match self.get_idlayer().uuid2rdn(e_uuid) {
                Ok(Some(idx_rdn)) => {
                    if rdn != idx_rdn {
                        admin_error!("Invalid uuid2rdn state -> incorrect idx rdn value");
                        return Err(ConsistencyError::BackendIndexSync);
                    }
                }
                r => {
                    admin_error!(state = ?r, "Invalid uuid2rdn state");
                    return Err(ConsistencyError::BackendIndexSync);
                }
            };
        }

        // Check the other entry:attr indexes are valid
        //
        // This is actually pretty hard to check, because we can check a value *should*
        // exist, but not that a value should NOT be present in the index. Thought needed ...

        // Got here? Ok!
        Ok(())
    }

    fn verify_indexes(&mut self) -> Vec<Result<(), ConsistencyError>> {
        let idl = IdList::AllIds;
        let entries = match self.get_idlayer().get_identry(&idl) {
            Ok(s) => s,
            Err(e) => {
                admin_error!(?e, "get_identry failure");
                return vec![Err(ConsistencyError::Unknown)];
            }
        };

        let r = entries.iter().try_for_each(|e| self.verify_entry_index(e));

        if r.is_err() {
            vec![r]
        } else {
            Vec::new()
        }
    }

    fn verify_ruv(&mut self, results: &mut Vec<Result<(), ConsistencyError>>) {
        // The way we verify this is building a whole second RUV and then comparing it.
        let idl = IdList::AllIds;
        let entries = match self.get_idlayer().get_identry(&idl) {
            Ok(ent) => ent,
            Err(e) => {
                results.push(Err(ConsistencyError::Unknown));
                admin_error!(?e, "get_identry failed");
                return;
            }
        };

        self.get_ruv().verify(&entries, results);
    }

    fn backup(&mut self, dst_path: &str) -> Result<(), OperationError> {
        // load all entries into RAM, may need to change this later
        // if the size of the database compared to RAM is an issue
        let idl = IdList::AllIds;
        let idlayer = self.get_idlayer();
        let raw_entries: Vec<IdRawEntry> = idlayer.get_identry_raw(&idl)?;

        let entries: Result<Vec<DbEntry>, _> = raw_entries
            .iter()
            .map(|id_ent| {
                serde_json::from_slice(id_ent.data.as_slice())
                    .map_err(|_| OperationError::SerdeJsonError) // log?
            })
            .collect();

        let entries = entries?;

        let db_s_uuid = idlayer
            .get_db_s_uuid()
            .and_then(|u| u.ok_or(OperationError::InvalidDbState))?;
        let db_d_uuid = idlayer
            .get_db_d_uuid()
            .and_then(|u| u.ok_or(OperationError::InvalidDbState))?;
        let db_ts_max = idlayer
            .get_db_ts_max()
            .and_then(|u| u.ok_or(OperationError::InvalidDbState))?;

        let bak = DbBackup::V2 {
            db_s_uuid,
            db_d_uuid,
            db_ts_max,
            entries,
        };

        let serialized_entries_str = serde_json::to_string(&bak).map_err(|e| {
            admin_error!(?e, "serde error");
            OperationError::SerdeJsonError
        })?;

        fs::write(dst_path, serialized_entries_str)
            .map(|_| ())
            .map_err(|e| {
                admin_error!(?e, "fs::write error");
                OperationError::FsError
            })
    }

    fn name2uuid(&mut self, name: &str) -> Result<Option<Uuid>, OperationError> {
        self.get_idlayer().name2uuid(name)
    }

    fn externalid2uuid(&mut self, name: &str) -> Result<Option<Uuid>, OperationError> {
        self.get_idlayer().externalid2uuid(name)
    }

    fn uuid2spn(&mut self, uuid: Uuid) -> Result<Option<Value>, OperationError> {
        self.get_idlayer().uuid2spn(uuid)
    }

    fn uuid2rdn(&mut self, uuid: Uuid) -> Result<Option<String>, OperationError> {
        self.get_idlayer().uuid2rdn(uuid)
    }
}

impl<'a> BackendTransaction for BackendReadTransaction<'a> {
    type IdlLayerType = IdlArcSqliteReadTransaction<'a>;
    type RuvType = ReplicationUpdateVectorReadTransaction<'a>;

    fn get_idlayer(&mut self) -> &mut IdlArcSqliteReadTransaction<'a> {
        &mut self.idlayer
    }

    fn get_ruv(&mut self) -> &mut ReplicationUpdateVectorReadTransaction<'a> {
        &mut self.ruv
    }

    fn get_idxmeta_ref(&self) -> &IdxMeta {
        &self.idxmeta
    }
}

impl<'a> BackendReadTransaction<'a> {
    pub fn list_indexes(&mut self) -> Result<Vec<String>, OperationError> {
        self.get_idlayer().list_idxs()
    }

    pub fn list_id2entry(&mut self) -> Result<Vec<(u64, String)>, OperationError> {
        self.get_idlayer().list_id2entry()
    }

    pub fn list_index_content(
        &mut self,
        index_name: &str,
    ) -> Result<Vec<(String, IDLBitRange)>, OperationError> {
        self.get_idlayer().list_index_content(index_name)
    }

    pub fn get_id2entry(&mut self, id: u64) -> Result<(u64, String), OperationError> {
        self.get_idlayer().get_id2entry(id)
    }
}

impl<'a> BackendTransaction for BackendWriteTransaction<'a> {
    type IdlLayerType = IdlArcSqliteWriteTransaction<'a>;
    type RuvType = ReplicationUpdateVectorWriteTransaction<'a>;

    fn get_idlayer(&mut self) -> &mut IdlArcSqliteWriteTransaction<'a> {
        &mut self.idlayer
    }

    fn get_ruv(&mut self) -> &mut ReplicationUpdateVectorWriteTransaction<'a> {
        &mut self.ruv
    }

    fn get_idxmeta_ref(&self) -> &IdxMeta {
        &self.idxmeta_wr
    }
}

impl<'a> BackendWriteTransaction<'a> {
    #[instrument(level = "debug", name = "be::create", skip_all)]
    pub fn create(
        &mut self,
        cid: &Cid,
        entries: Vec<Entry<EntrySealed, EntryNew>>,
    ) -> Result<Vec<Entry<EntrySealed, EntryCommitted>>, OperationError> {
        if entries.is_empty() {
            admin_error!("No entries provided to BE to create, invalid server call!");
            return Err(OperationError::EmptyRequest);
        }

        // Check that every entry has a change associated
        // that matches the cid?
        entries.iter().try_for_each(|e| {
            if e.get_changestate().contains_tail_cid(cid) {
                Ok(())
            } else {
                admin_error!(
                    "Entry changelog does not contain a change related to this transaction"
                );
                Err(OperationError::ReplEntryNotChanged)
            }
        })?;

        // Now, assign id's to all the new entries.

        let mut id_max = self.idlayer.get_id2entry_max_id()?;
        let c_entries: Vec<_> = entries
            .into_iter()
            .map(|e| {
                id_max += 1;
                e.into_sealed_committed_id(id_max)
            })
            .collect();

        // All good, lets update the RUV.
        // This auto compresses.
        let ruv_idl = IDLBitRange::from_iter(c_entries.iter().map(|e| e.get_id()));

        self.get_ruv().insert_change(cid, ruv_idl)?;

        self.idlayer.write_identries(c_entries.iter())?;

        self.idlayer.set_id2entry_max_id(id_max);

        // Now update the indexes as required.
        for e in c_entries.iter() {
            self.entry_index(None, Some(e))?
        }

        Ok(c_entries)
    }

    #[instrument(level = "debug", name = "be::create", skip_all)]
    /// This is similar to create, but used in the replication path as it skips the
    /// modification of the RUV and the checking of CIDs since these actions are not
    /// required during a replication refresh (else we'd create an infinite replication
    /// loop.)
    pub fn refresh(
        &mut self,
        entries: Vec<Entry<EntrySealed, EntryNew>>,
    ) -> Result<Vec<Entry<EntrySealed, EntryCommitted>>, OperationError> {
        if entries.is_empty() {
            admin_error!("No entries provided to BE to create, invalid server call!");
            return Err(OperationError::EmptyRequest);
        }

        // Assign id's to all the new entries.
        let mut id_max = self.idlayer.get_id2entry_max_id()?;
        let c_entries: Vec<_> = entries
            .into_iter()
            .map(|e| {
                id_max += 1;
                e.into_sealed_committed_id(id_max)
            })
            .collect();

        self.idlayer.write_identries(c_entries.iter())?;

        self.idlayer.set_id2entry_max_id(id_max);

        // Now update the indexes as required.
        for e in c_entries.iter() {
            self.entry_index(None, Some(e))?
        }

        Ok(c_entries)
    }

    #[instrument(level = "debug", name = "be::modify", skip_all)]
    pub fn modify(
        &mut self,
        cid: &Cid,
        pre_entries: &[Arc<EntrySealedCommitted>],
        post_entries: &[EntrySealedCommitted],
    ) -> Result<(), OperationError> {
        if post_entries.is_empty() || pre_entries.is_empty() {
            admin_error!("No entries provided to BE to modify, invalid server call!");
            return Err(OperationError::EmptyRequest);
        }

        assert!(post_entries.len() == pre_entries.len());

        post_entries.iter().try_for_each(|e| {
            if e.get_changestate().contains_tail_cid(cid) {
                Ok(())
            } else {
                admin_error!(
                    "Entry changelog does not contain a change related to this transaction"
                );
                Err(OperationError::ReplEntryNotChanged)
            }
        })?;

        // All good, lets update the RUV.
        // This auto compresses.
        let ruv_idl = IDLBitRange::from_iter(post_entries.iter().map(|e| e.get_id()));
        self.get_ruv().insert_change(cid, ruv_idl)?;

        // Now, given the list of id's, update them
        self.get_idlayer().write_identries(post_entries.iter())?;

        // Finally, we now reindex all the changed entries. We do this by iterating and zipping
        // over the set, because we know the list is in the same order.
        pre_entries
            .iter()
            .zip(post_entries.iter())
            .try_for_each(|(pre, post)| self.entry_index(Some(pre.as_ref()), Some(post)))
    }

    #[instrument(level = "debug", name = "be::reap_tombstones", skip_all)]
    pub fn reap_tombstones(&mut self, cid: &Cid) -> Result<usize, OperationError> {
        // We plan to clear the RUV up to this cid. So we need to build an IDL
        // of all the entries we need to examine.
        let idl = self.get_ruv().trim_up_to(cid).map_err(|e| {
            admin_error!(?e, "failed to trim RUV to {:?}", cid);
            e
        })?;

        let entries = self
            .get_idlayer()
            .get_identry(&IdList::Indexed(idl))
            .map_err(|e| {
                admin_error!(?e, "get_identry failed");
                e
            })?;

        if entries.is_empty() {
            admin_info!("No entries affected - reap_tombstones operation success");
            return Ok(0);
        }

        // Now that we have a list of entries we need to partition them into
        // two sets. The entries that are tombstoned and ready to reap_tombstones, and
        // the entries that need to have their change logs trimmed.
        //
        // Remember, these tombstones can be reaped because they were tombstoned at time
        // point 'cid', and since we are now "past" that minimum cid, then other servers
        // will also be trimming these out.
        //
        // Note unlike a changelog impl, we don't need to trim changestates here. We
        // only need the RUV trimmed so that we know if other servers are laggin behind!

        // What entries are tombstones and ready to be deleted?

        let (tombstones, leftover): (Vec<_>, Vec<_>) = entries
            .into_iter()
            .partition(|e| e.get_changestate().can_delete(cid));

        let ruv_idls = self.get_ruv().ruv_idls();

        // Assert that anything leftover still either is *alive* OR is a tombstone
        // and has entries in the RUV!

        if !leftover
            .iter()
            .all(|e| e.get_changestate().is_live() || ruv_idls.contains(e.get_id()))
        {
            admin_error!("Left over entries may be orphaned due to missing RUV entries");
            return Err(OperationError::ReplInvalidRUVState);
        }

        // Now setup to reap_tombstones the tombstones. Remember, in the post cleanup, it's could
        // now have been trimmed to a point we can purge them!

        // Assert the id's exist on the entry.
        let id_list: IDLBitRange = tombstones.iter().map(|e| e.get_id()).collect();

        // Ensure nothing here exists in the RUV index, else it means
        // we didn't trim properly, or some other state violation has occurred.
        if !((&ruv_idls & &id_list).is_empty()) {
            admin_error!("RUV still contains entries that are going to be removed.");
            return Err(OperationError::ReplInvalidRUVState);
        }

        // Now, given the list of id's, reap_tombstones them.
        let sz = id_list.len();
        self.get_idlayer().delete_identry(id_list.into_iter())?;

        // Finally, purge the indexes from the entries we removed. These still have
        // indexes due to class=tombstone.
        tombstones
            .iter()
            .try_for_each(|e| self.entry_index(Some(e), None))?;

        Ok(sz)
    }

    #[instrument(level = "debug", name = "be::update_idxmeta", skip_all)]
    pub fn update_idxmeta(&mut self, idxkeys: Vec<IdxKey>) -> Result<(), OperationError> {
        if self.is_idx_slopeyness_generated()? {
            trace!("Indexing slopes available");
        } else {
            admin_warn!(
                "No indexing slopes available. You should consider reindexing to generate these"
            );
        };

        // Setup idxkeys here. By default we set these all to "max slope" aka
        // all indexes are "equal" but also worse case unless analysed. If they
        // have been analysed, we can set the slope factor into here.
        let idxkeys: Result<Map<_, _>, _> = idxkeys
            .into_iter()
            .map(|k| self.get_idx_slope(&k).map(|slope| (k, slope)))
            .collect();

        let mut idxkeys = idxkeys?;

        std::mem::swap(&mut self.idxmeta_wr.deref_mut().idxkeys, &mut idxkeys);
        Ok(())
    }

    // Should take a mut index set, and then we write the whole thing back
    // in a single stripe.
    //
    // So we need a cache, which we load indexes into as we do ops, then we
    // modify them.
    //
    // At the end, we flush those cchange outs in a single run.
    // For create this is probably a
    // TODO: Can this be improved?
    #[allow(clippy::cognitive_complexity)]
    fn entry_index(
        &mut self,
        pre: Option<&Entry<EntrySealed, EntryCommitted>>,
        post: Option<&Entry<EntrySealed, EntryCommitted>>,
    ) -> Result<(), OperationError> {
        let (e_uuid, e_id, uuid_same) = match (pre, post) {
            (None, None) => {
                admin_error!("Invalid call to entry_index - no entries provided");
                return Err(OperationError::InvalidState);
            }
            (Some(pre), None) => {
                trace!("Attempting to remove entry indexes");
                (pre.get_uuid(), pre.get_id(), true)
            }
            (None, Some(post)) => {
                trace!("Attempting to create entry indexes");
                (post.get_uuid(), post.get_id(), true)
            }
            (Some(pre), Some(post)) => {
                trace!("Attempting to modify entry indexes");
                assert!(pre.get_id() == post.get_id());
                (
                    post.get_uuid(),
                    post.get_id(),
                    pre.get_uuid() == post.get_uuid(),
                )
            }
        };

        // Update the names/uuid maps. These have to mask out entries
        // that are recycled or tombstones, so these pretend as "deleted"
        // and can trigger correct actions.
        //

        let mask_pre = pre.and_then(|e| e.mask_recycled_ts());
        let mask_pre = if !uuid_same {
            // Okay, so if the uuids are different this is probably from
            // a replication conflict.  We can't just use the normal none/some
            // check from the Entry::idx functions as they only yield partial
            // changes. Because the uuid is changing, we have to treat pre
            // as a deleting entry, regardless of what state post is in.
            let uuid = mask_pre.map(|e| e.get_uuid()).ok_or_else(|| {
                admin_error!("Invalid entry state - possible memory corruption");
                OperationError::InvalidState
            })?;

            let (n2u_add, n2u_rem) = Entry::idx_name2uuid_diff(mask_pre, None);
            // There will never be content to add.
            assert!(n2u_add.is_none());

            let (eid2u_add, eid2u_rem) = Entry::idx_externalid2uuid_diff(mask_pre, None);
            // There will never be content to add.
            assert!(eid2u_add.is_none());

            let u2s_act = Entry::idx_uuid2spn_diff(mask_pre, None);
            let u2r_act = Entry::idx_uuid2rdn_diff(mask_pre, None);

            trace!(?n2u_rem, ?eid2u_rem, ?u2s_act, ?u2r_act,);

            // Write the changes out to the backend
            if let Some(rem) = n2u_rem {
                self.idlayer.write_name2uuid_rem(rem)?
            }

            if let Some(rem) = eid2u_rem {
                self.idlayer.write_externalid2uuid_rem(rem)?
            }

            match u2s_act {
                None => {}
                Some(Ok(k)) => self.idlayer.write_uuid2spn(uuid, Some(k))?,
                Some(Err(_)) => self.idlayer.write_uuid2spn(uuid, None)?,
            }

            match u2r_act {
                None => {}
                Some(Ok(k)) => self.idlayer.write_uuid2rdn(uuid, Some(k))?,
                Some(Err(_)) => self.idlayer.write_uuid2rdn(uuid, None)?,
            }
            // Return none, mask_pre is now completed.
            None
        } else {
            // Return the state.
            mask_pre
        };

        let mask_post = post.and_then(|e| e.mask_recycled_ts());
        let (n2u_add, n2u_rem) = Entry::idx_name2uuid_diff(mask_pre, mask_post);
        let (eid2u_add, eid2u_rem) = Entry::idx_externalid2uuid_diff(mask_pre, mask_post);

        let u2s_act = Entry::idx_uuid2spn_diff(mask_pre, mask_post);
        let u2r_act = Entry::idx_uuid2rdn_diff(mask_pre, mask_post);

        trace!(
            ?n2u_add,
            ?n2u_rem,
            ?eid2u_add,
            ?eid2u_rem,
            ?u2s_act,
            ?u2r_act
        );

        // Write the changes out to the backend
        if let Some(add) = n2u_add {
            self.idlayer.write_name2uuid_add(e_uuid, add)?
        }
        if let Some(rem) = n2u_rem {
            self.idlayer.write_name2uuid_rem(rem)?
        }

        if let Some(add) = eid2u_add {
            self.idlayer.write_externalid2uuid_add(e_uuid, add)?
        }
        if let Some(rem) = eid2u_rem {
            self.idlayer.write_externalid2uuid_rem(rem)?
        }

        match u2s_act {
            None => {}
            Some(Ok(k)) => self.idlayer.write_uuid2spn(e_uuid, Some(k))?,
            Some(Err(_)) => self.idlayer.write_uuid2spn(e_uuid, None)?,
        }

        match u2r_act {
            None => {}
            Some(Ok(k)) => self.idlayer.write_uuid2rdn(e_uuid, Some(k))?,
            Some(Err(_)) => self.idlayer.write_uuid2rdn(e_uuid, None)?,
        }

        // Extremely Cursed - Okay, we know that self.idxmeta will NOT be changed
        // in this function, but we need to borrow self as mut for the caches in
        // get_idl to work. As a result, this causes a double borrow. To work around
        // this we discard the lifetime on idxmeta, because we know that it will
        // remain constant for the life of the operation.

        let idxmeta = unsafe { &(*(&self.idxmeta_wr.idxkeys as *const _)) };

        let idx_diff = Entry::idx_diff(idxmeta, pre, post);

        idx_diff.into_iter()
            .try_for_each(|act| {
                match act {
                    Ok((attr, itype, idx_key)) => {
                        trace!("Adding {:?} idx -> {:?}: {:?}", itype, attr, idx_key);
                        match self.idlayer.get_idl(attr, itype, &idx_key)? {
                            Some(mut idl) => {
                                idl.insert_id(e_id);
                                self.idlayer.write_idl(attr, itype, &idx_key, &idl)
                            }
                            None => {
                                warn!(
                                    "WARNING: index {:?} {:?} was not found. YOU MUST REINDEX YOUR DATABASE",
                                    attr, itype
                                );
                                Ok(())
                            }
                        }
                    }
                    Err((attr, itype, idx_key)) => {
                        trace!("Removing {:?} idx -> {:?}: {:?}", itype, attr, idx_key);
                        match self.idlayer.get_idl(attr, itype, &idx_key)? {
                            Some(mut idl) => {
                                idl.remove_id(e_id);
                                self.idlayer.write_idl(attr, itype, &idx_key, &idl)
                            }
                            None => {
                                warn!(
                                    "WARNING: index {:?} {:?} was not found. YOU MUST REINDEX YOUR DATABASE",
                                    attr, itype
                                );
                                Ok(())
                            }
                        }
                    }
                }
            })
        // End try_for_each
    }

    #[allow(dead_code)]
    fn missing_idxs(&mut self) -> Result<Vec<(AttrString, IndexType)>, OperationError> {
        let idx_table_list = self.get_idlayer().list_idxs()?;

        // Turn the vec to a real set
        let idx_table_set: HashSet<_> = idx_table_list.into_iter().collect();

        let missing: Vec<_> = self
            .idxmeta_wr
            .idxkeys
            .keys()
            .filter_map(|ikey| {
                // what would the table name be?
                let tname = format!("idx_{}_{}", ikey.itype.as_idx_str(), ikey.attr.as_str());
                trace!("Checking for {}", tname);

                if idx_table_set.contains(&tname) {
                    None
                } else {
                    Some((ikey.attr.clone(), ikey.itype))
                }
            })
            .collect();
        Ok(missing)
    }

    fn create_idxs(&mut self) -> Result<(), OperationError> {
        // Create name2uuid and uuid2name
        trace!("Creating index -> name2uuid");
        self.idlayer.create_name2uuid()?;

        trace!("Creating index -> externalid2uuid");
        self.idlayer.create_externalid2uuid()?;

        trace!("Creating index -> uuid2spn");
        self.idlayer.create_uuid2spn()?;

        trace!("Creating index -> uuid2rdn");
        self.idlayer.create_uuid2rdn()?;

        self.idxmeta_wr
            .idxkeys
            .keys()
            .try_for_each(|ikey| self.idlayer.create_idx(&ikey.attr, ikey.itype))
    }

    pub fn upgrade_reindex(&mut self, v: i64) -> Result<(), OperationError> {
        let dbv = self.get_db_index_version();
        admin_debug!(?dbv, ?v, "upgrade_reindex");
        if dbv < v {
            limmediate_warning!(
                "NOTICE: A system reindex is required. This may take a long time ...\n"
            );
            self.reindex()?;
            limmediate_warning!("NOTICE: System reindex complete\n");
            self.set_db_index_version(v)
        } else {
            Ok(())
        }
    }

    pub fn reindex(&mut self) -> Result<(), OperationError> {
        // Purge the idxs
        unsafe { self.idlayer.purge_idxs()? };

        // Using the index metadata on the txn, create all our idx tables
        self.create_idxs()?;

        // Now, we need to iterate over everything in id2entry and index them
        // Future idea: Do this in batches of X amount to limit memory
        // consumption.
        let idl = IdList::AllIds;
        let entries = self.idlayer.get_identry(&idl).map_err(|e| {
            admin_error!(err = ?e, "get_identry failure");
            e
        })?;

        let mut count = 0;

        entries
            .iter()
            .try_for_each(|e| {
                count += 1;
                if count % 2500 == 0 {
                    limmediate_warning!("{}", count);
                } else if count % 250 == 0 {
                    limmediate_warning!(".");
                }
                self.entry_index(None, Some(e))
            })
            .map_err(|e| {
                admin_error!("reindex failed -> {:?}", e);
                e
            })?;
        limmediate_warning!(" reindexed {} entries ✅\n", count);
        limmediate_warning!("Optimising Indexes ... ");
        self.idlayer.optimise_dirty_idls();
        limmediate_warning!("done ✅\n");
        limmediate_warning!("Calculating Index Optimisation Slopes ... ");
        self.idlayer.analyse_idx_slopes().map_err(|e| {
            admin_error!(err = ?e, "index optimisation failed");
            e
        })?;
        limmediate_warning!("done ✅\n");
        Ok(())
    }

    fn purge_idxs(&mut self) -> Result<(), OperationError> {
        unsafe { self.get_idlayer().purge_idxs() }
    }

    pub(crate) fn danger_delete_all_db_content(&mut self) -> Result<(), OperationError> {
        unsafe {
            self.get_idlayer()
                .purge_id2entry()
                .and_then(|_| self.purge_idxs())
        }
    }

    #[cfg(test)]
    pub fn load_test_idl(
        &mut self,
        attr: &String,
        itype: IndexType,
        idx_key: &String,
    ) -> Result<Option<IDLBitRange>, OperationError> {
        self.get_idlayer().get_idl(attr, itype, idx_key)
    }

    fn is_idx_slopeyness_generated(&mut self) -> Result<bool, OperationError> {
        self.get_idlayer().is_idx_slopeyness_generated()
    }

    fn get_idx_slope(&mut self, ikey: &IdxKey) -> Result<IdxSlope, OperationError> {
        // Do we have the slopeyness?
        let slope = self
            .get_idlayer()
            .get_idx_slope(ikey)?
            .unwrap_or_else(|| get_idx_slope_default(ikey));
        trace!("index slope - {:?} -> {:?}", ikey, slope);
        Ok(slope)
    }

    pub fn restore(&mut self, src_path: &str) -> Result<(), OperationError> {
        let idlayer = self.get_idlayer();
        // load all entries into RAM, may need to change this later
        // if the size of the database compared to RAM is an issue
        let serialized_string = fs::read_to_string(src_path).map_err(|e| {
            admin_error!("fs::read_to_string {:?}", e);
            OperationError::FsError
        })?;

        unsafe { idlayer.purge_id2entry() }.map_err(|e| {
            admin_error!("purge_id2entry failed {:?}", e);
            e
        })?;

        let dbbak_option: Result<DbBackup, serde_json::Error> =
            serde_json::from_str(&serialized_string);

        let dbbak = dbbak_option.map_err(|e| {
            admin_error!("serde_json error {:?}", e);
            OperationError::SerdeJsonError
        })?;

        let dbentries = match dbbak {
            DbBackup::V1(dbentries) => dbentries,
            DbBackup::V2 {
                db_s_uuid,
                db_d_uuid,
                db_ts_max,
                entries,
            } => {
                // Do stuff.
                idlayer.write_db_s_uuid(db_s_uuid)?;
                idlayer.write_db_d_uuid(db_d_uuid)?;
                idlayer.set_db_ts_max(db_ts_max)?;
                entries
            }
        };

        info!("Restoring {} entries ...", dbentries.len());

        // Migrate any v1 entries to v2 if needed.
        let dbentries = dbentries
            .into_iter()
            .map(|dbe| dbe.convert_to_v2())
            .collect::<Result<Vec<_>, _>>()?;

        // Now, we setup all the entries with new ids.
        let mut id_max = 0;
        let identries: Result<Vec<IdRawEntry>, _> = dbentries
            .iter()
            .map(|e| {
                id_max += 1;
                let data = serde_json::to_vec(&e).map_err(|_| OperationError::SerdeCborError)?;
                Ok(IdRawEntry { id: id_max, data })
            })
            .collect();

        idlayer.write_identries_raw(identries?.into_iter())?;

        info!("Restored {} entries", dbentries.len());

        // Reindex now we are loaded.
        self.reindex()?;

        let vr = self.verify();
        if vr.is_empty() {
            Ok(())
        } else {
            Err(OperationError::ConsistencyError(vr))
        }
    }

    #[instrument(level = "debug", name = "be::ruv_rebuild", skip_all)]
    pub fn ruv_rebuild(&mut self) -> Result<(), OperationError> {
        // Rebuild the ruv!
        // For now this has to read from all the entries in the DB, but in the future
        // we'll actually store this properly (?). If it turns out this is really fast
        // we may just rebuild this always on startup.

        // NOTE: An important detail is that we don't rely on indexes here!

        let idl = IdList::AllIds;
        let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
            admin_error!(?e, "get_identry failed");
            e
        })?;

        self.get_ruv().rebuild(&entries)?;

        Ok(())
    }

    pub fn commit(self) -> Result<(), OperationError> {
        let BackendWriteTransaction {
            idlayer,
            idxmeta_wr,
            ruv,
        } = self;

        idlayer.commit().map(|()| {
            ruv.commit();
            idxmeta_wr.commit();
        })
    }

    fn reset_db_s_uuid(&mut self) -> Result<Uuid, OperationError> {
        // The value is missing. Generate a new one and store it.
        let nsid = Uuid::new_v4();
        self.get_idlayer().write_db_s_uuid(nsid)?;
        Ok(nsid)
    }

    pub fn get_db_s_uuid(&mut self) -> Uuid {
        #[allow(clippy::expect_used)]
        match self
            .get_idlayer()
            .get_db_s_uuid()
            .expect("DBLayer Error!!!")
        {
            Some(s_uuid) => s_uuid,
            None => self.reset_db_s_uuid().expect("Failed to regenerate S_UUID"),
        }
    }

    /// This generates a new domain UUID and stores it into the database,
    /// returning the new UUID
    fn reset_db_d_uuid(&mut self) -> Result<Uuid, OperationError> {
        let nsid = Uuid::new_v4();
        self.get_idlayer().write_db_d_uuid(nsid)?;
        Ok(nsid)
    }

    /// Manually set a new domain UUID and store it into the DB. This is used
    /// as part of a replication refresh.
    pub fn set_db_d_uuid(&mut self, nsid: Uuid) -> Result<(), OperationError> {
        self.get_idlayer().write_db_d_uuid(nsid)
    }

    /// This pulls the domain UUID from the database
    pub fn get_db_d_uuid(&mut self) -> Uuid {
        #[allow(clippy::expect_used)]
        match self
            .get_idlayer()
            .get_db_d_uuid()
            .expect("DBLayer Error retrieving Domain UUID!!!")
        {
            Some(d_uuid) => d_uuid,
            None => self.reset_db_d_uuid().expect("Failed to regenerate D_UUID"),
        }
    }

    pub fn set_db_ts_max(&mut self, ts: Duration) -> Result<(), OperationError> {
        self.get_idlayer().set_db_ts_max(ts)
    }

    pub fn get_db_ts_max(&mut self, ts: Duration) -> Result<Duration, OperationError> {
        // if none, return ts. If found, return it.
        match self.get_idlayer().get_db_ts_max()? {
            Some(dts) => Ok(dts),
            None => Ok(ts),
        }
    }

    fn get_db_index_version(&mut self) -> i64 {
        self.get_idlayer().get_db_index_version()
    }

    fn set_db_index_version(&mut self, v: i64) -> Result<(), OperationError> {
        self.get_idlayer().set_db_index_version(v)
    }
}

// We have a number of hardcoded, "obvious" slopes that should
// exist. We return these when the analysis has not been run, as
// these are values that are generally "good enough" for most applications
fn get_idx_slope_default(ikey: &IdxKey) -> IdxSlope {
    match (ikey.attr.as_str(), &ikey.itype) {
        ("name", IndexType::Equality)
        | ("spn", IndexType::Equality)
        | ("uuid", IndexType::Equality) => 1,
        ("class", IndexType::Equality) => 180,
        (_, IndexType::Equality) => 45,
        (_, IndexType::SubString) => 90,
        (_, IndexType::Presence) => 90,
    }
}

// In the future this will do the routing between the chosen backends etc.
impl Backend {
    #[instrument(level = "debug", name = "be::new", skip_all)]
    pub fn new(
        mut cfg: BackendConfig,
        // path: &str,
        // mut pool_size: u32,
        // fstype: FsType,
        idxkeys: Vec<IdxKey>,
        vacuum: bool,
    ) -> Result<Self, OperationError> {
        info!("DB tickets -> {:?}", cfg.pool_size);
        info!("Profile -> {}", env!("KANIDM_PROFILE_NAME"));
        info!("CPU Flags -> {}", env!("KANIDM_CPU_FLAGS"));

        // If in memory, reduce pool to 1
        if cfg.path.is_empty() {
            cfg.pool_size = 1;
        }

        // Setup idxkeys here. By default we set these all to "max slope" aka
        // all indexes are "equal" but also worse case unless analysed.
        //
        // During startup this will be "fixed" as the schema core will call reload_idxmeta
        // which will trigger a reload of the analysis data (if present).
        let idxkeys: Map<_, _> = idxkeys
            .into_iter()
            .map(|ikey| {
                let slope = get_idx_slope_default(&ikey);
                (ikey, slope)
            })
            .collect();

        // RUV-TODO
        // Load the replication update vector here. For now we rebuild every startup
        // from the database.
        let ruv = Arc::new(ReplicationUpdateVector::default());

        // this has a ::memory() type, but will path == "" work?
        let idlayer = Arc::new(IdlArcSqlite::new(&cfg, vacuum)?);
        let be = Backend {
            cfg,
            idlayer,
            ruv,
            idxmeta: Arc::new(CowCell::new(IdxMeta::new(idxkeys))),
        };

        // Now complete our setup with a txn
        // In this case we can use an empty idx meta because we don't
        // access any parts of
        // the indexing subsystem here.
        let mut idl_write = be.idlayer.write();
        idl_write
            .setup()
            .and_then(|_| idl_write.commit())
            .map_err(|e| {
                admin_error!(?e, "Failed to setup idlayer");
                e
            })?;

        // Now rebuild the ruv.
        let mut be_write = be.write();
        be_write
            .ruv_rebuild()
            .and_then(|_| be_write.commit())
            .map_err(|e| {
                admin_error!(?e, "Failed to reload ruv");
                e
            })?;

        Ok(be)
    }

    pub fn get_pool_size(&self) -> u32 {
        debug_assert!(self.cfg.pool_size > 0);
        self.cfg.pool_size
    }

    pub fn try_quiesce(&self) {
        self.idlayer.try_quiesce();
    }

    pub fn read(&self) -> BackendReadTransaction {
        BackendReadTransaction {
            idlayer: self.idlayer.read(),
            idxmeta: self.idxmeta.read(),
            ruv: self.ruv.read(),
        }
    }

    pub fn write(&self) -> BackendWriteTransaction {
        BackendWriteTransaction {
            idlayer: self.idlayer.write(),
            idxmeta_wr: self.idxmeta.write(),
            ruv: self.ruv.write(),
        }
    }

    // Should this actually call the idlayer directly?
    pub fn reset_db_s_uuid(&self) -> Uuid {
        let mut wr = self.write();
        #[allow(clippy::expect_used)]
        let sid = wr
            .reset_db_s_uuid()
            .expect("unable to reset db server uuid");
        #[allow(clippy::expect_used)]
        wr.commit()
            .expect("Unable to commit to backend, can not proceed");
        sid
    }

    /*
    pub fn get_db_s_uuid(&self) -> Uuid {
        let wr = self.write(Set::new());
        wr.reset_db_s_uuid().unwrap()
    }
    */
}

// What are the possible actions we'll receive here?

#[cfg(test)]
mod tests {
    use std::fs;
    use std::iter::FromIterator;
    use std::sync::Arc;
    use std::time::Duration;

    use idlset::v2::IDLBitRange;

    use super::super::entry::{Entry, EntryInit, EntryNew};
    use super::Limits;
    use super::{
        Backend, BackendConfig, BackendTransaction, BackendWriteTransaction, DbBackup, IdList,
        IdxKey, OperationError,
    };
    use crate::prelude::*;
    use crate::repl::cid::Cid;
    use crate::value::{IndexType, PartialValue, Value};

    lazy_static! {
        static ref CID_ZERO: Cid = unsafe { Cid::new_zero() };
        static ref CID_ONE: Cid = unsafe { Cid::new_count(1) };
        static ref CID_TWO: Cid = unsafe { Cid::new_count(2) };
        static ref CID_THREE: Cid = unsafe { Cid::new_count(3) };
    }

    macro_rules! run_test {
        ($test_fn:expr) => {{
            let _ = sketching::test_init();

            // This is a demo idxmeta, purely for testing.
            let idxmeta = vec![
                IdxKey {
                    attr: AttrString::from("name"),
                    itype: IndexType::Equality,
                },
                IdxKey {
                    attr: AttrString::from("name"),
                    itype: IndexType::Presence,
                },
                IdxKey {
                    attr: AttrString::from("name"),
                    itype: IndexType::SubString,
                },
                IdxKey {
                    attr: AttrString::from("uuid"),
                    itype: IndexType::Equality,
                },
                IdxKey {
                    attr: AttrString::from("uuid"),
                    itype: IndexType::Presence,
                },
                IdxKey {
                    attr: AttrString::from("ta"),
                    itype: IndexType::Equality,
                },
                IdxKey {
                    attr: AttrString::from("tb"),
                    itype: IndexType::Equality,
                },
            ];

            let be = Backend::new(BackendConfig::new_test("main"), idxmeta, false)
                .expect("Failed to setup backend");

            let mut be_txn = be.write();

            let r = $test_fn(&mut be_txn);
            // Commit, to guarantee it worked.
            assert!(be_txn.commit().is_ok());
            r
        }};
    }

    macro_rules! entry_exists {
        ($be:expr, $ent:expr) => {{
            let ei = unsafe { $ent.clone().into_sealed_committed() };
            let filt = unsafe {
                ei.filter_from_attrs(&vec![AttrString::from("uuid")])
                    .expect("failed to generate filter")
                    .into_valid_resolved()
            };
            let lims = Limits::unlimited();
            let entries = $be.search(&lims, &filt).expect("failed to search");
            entries.first().is_some()
        }};
    }

    macro_rules! entry_attr_pres {
        ($be:expr, $ent:expr, $attr:expr) => {{
            let ei = unsafe { $ent.clone().into_sealed_committed() };
            let filt = unsafe {
                ei.filter_from_attrs(&vec![AttrString::from("userid")])
                    .expect("failed to generate filter")
                    .into_valid_resolved()
            };
            let lims = Limits::unlimited();
            let entries = $be.search(&lims, &filt).expect("failed to search");
            match entries.first() {
                Some(ent) => ent.attribute_pres($attr),
                None => false,
            }
        }};
    }

    macro_rules! idl_state {
        ($be:expr, $attr:expr, $itype:expr, $idx_key:expr, $expect:expr) => {{
            let t_idl = $be
                .load_test_idl(&$attr.to_string(), $itype, &$idx_key.to_string())
                .expect("IdList Load failed");
            let t = $expect.map(|v: Vec<u64>| IDLBitRange::from_iter(v));
            assert_eq!(t_idl, t);
        }};
    }

    #[test]
    fn test_be_simple_create() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Create");

            let empty_result = be.create(&CID_ZERO, Vec::new());
            trace!("{:?}", empty_result);
            assert_eq!(empty_result, Err(OperationError::EmptyRequest));

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava("userid", Value::from("william"));
            e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            let e = unsafe { e.into_sealed_new() };

            let single_result = be.create(&CID_ZERO, vec![e.clone()]);

            assert!(single_result.is_ok());

            // Construct a filter
            assert!(entry_exists!(be, e));
        });
    }

    #[test]
    fn test_be_simple_search() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Search");

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava("userid", Value::from("claire"));
            e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            let e = unsafe { e.into_sealed_new() };

            let single_result = be.create(&CID_ZERO, vec![e]);
            assert!(single_result.is_ok());
            // Test a simple EQ search

            let filt =
                unsafe { filter_resolved!(f_eq("userid", PartialValue::new_utf8s("claire"))) };

            let lims = Limits::unlimited();

            let r = be.search(&lims, &filt);
            assert!(r.expect("Search failed!").len() == 1);

            // Test empty search

            // Test class pres

            // Search with no results
        });
    }

    #[test]
    fn test_be_simple_modify() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Modify");
            let lims = Limits::unlimited();
            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("userid", Value::from("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("userid", Value::from("alice"));
            e2.add_ava("uuid", Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"));

            let ve1 = unsafe { e1.clone().into_sealed_new() };
            let ve2 = unsafe { e2.clone().into_sealed_new() };

            assert!(be.create(&CID_ZERO, vec![ve1, ve2]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));

            // You need to now retrieve the entries back out to get the entry id's
            let mut results = be
                .search(&lims, unsafe { &filter_resolved!(f_pres("userid")) })
                .expect("Failed to search");

            // Get these out to usable entries.
            let r1 = results.remove(0);
            let r2 = results.remove(0);

            let mut r1 = unsafe { r1.as_ref().clone().into_invalid() };
            let mut r2 = unsafe { r2.as_ref().clone().into_invalid() };

            // Modify no id (err)
            // This is now impossible due to the state machine design.
            // However, with some unsafe ....
            let ue1 = unsafe { e1.clone().into_sealed_committed() };
            assert!(be
                .modify(&CID_ZERO, &[Arc::new(ue1.clone())], &[ue1])
                .is_err());
            // Modify none
            assert!(be.modify(&CID_ZERO, &[], &[]).is_err());

            // Make some changes to r1, r2.
            let pre1 = unsafe { Arc::new(r1.clone().into_sealed_committed()) };
            let pre2 = unsafe { Arc::new(r2.clone().into_sealed_committed()) };
            r1.add_ava("desc", Value::from("modified"));
            r2.add_ava("desc", Value::from("modified"));

            // Now ... cheat.

            let vr1 = unsafe { r1.into_sealed_committed() };
            let vr2 = unsafe { r2.into_sealed_committed() };

            // Modify single
            assert!(be.modify(&CID_ZERO, &[pre1], &[vr1.clone()]).is_ok());
            // Assert no other changes
            assert!(entry_attr_pres!(be, vr1, "desc"));
            assert!(!entry_attr_pres!(be, vr2, "desc"));

            // Modify both
            assert!(be
                .modify(
                    &CID_ZERO,
                    &[Arc::new(vr1.clone()), pre2],
                    &[vr1.clone(), vr2.clone()]
                )
                .is_ok());

            assert!(entry_attr_pres!(be, vr1, "desc"));
            assert!(entry_attr_pres!(be, vr2, "desc"));
        });
    }

    #[test]
    fn test_be_simple_delete() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Delete");
            let lims = Limits::unlimited();

            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("userid", Value::from("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("userid", Value::from("alice"));
            e2.add_ava("uuid", Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"));

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava("userid", Value::from("lucy"));
            e3.add_ava("uuid", Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"));

            let ve1 = unsafe { e1.clone().into_sealed_new() };
            let ve2 = unsafe { e2.clone().into_sealed_new() };
            let ve3 = unsafe { e3.clone().into_sealed_new() };

            assert!(be.create(&CID_ZERO, vec![ve1, ve2, ve3]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));
            assert!(entry_exists!(be, e3));

            // You need to now retrieve the entries back out to get the entry id's
            let mut results = be
                .search(&lims, unsafe { &filter_resolved!(f_pres("userid")) })
                .expect("Failed to search");

            // Get these out to usable entries.
            let r1 = results.remove(0);
            let r2 = results.remove(0);
            let r3 = results.remove(0);

            // Deletes nothing, all entries are live.
            assert!(matches!(be.reap_tombstones(&CID_ZERO), Ok(0)));

            // Put them into the tombstone state, and write that down.
            // This sets up the RUV with the changes.
            let r1_ts = unsafe { r1.to_tombstone(CID_ONE.clone()).into_sealed_committed() };

            assert!(be.modify(&CID_ONE, &[r1], &[r1_ts.clone()]).is_ok());

            let r2_ts = unsafe { r2.to_tombstone(CID_TWO.clone()).into_sealed_committed() };
            let r3_ts = unsafe { r3.to_tombstone(CID_TWO.clone()).into_sealed_committed() };

            assert!(be
                .modify(&CID_TWO, &[r2, r3], &[r2_ts.clone(), r3_ts.clone()])
                .is_ok());

            // The entry are now tombstones, but is still in the ruv. This is because we
            // targeted CID_ZERO, not ONE.
            assert!(matches!(be.reap_tombstones(&CID_ZERO), Ok(0)));

            assert!(entry_exists!(be, r1_ts));
            assert!(entry_exists!(be, r2_ts));
            assert!(entry_exists!(be, r3_ts));

            assert!(matches!(be.reap_tombstones(&CID_ONE), Ok(0)));

            assert!(entry_exists!(be, r1_ts));
            assert!(entry_exists!(be, r2_ts));
            assert!(entry_exists!(be, r3_ts));

            assert!(matches!(be.reap_tombstones(&CID_TWO), Ok(1)));

            assert!(!entry_exists!(be, r1_ts));
            assert!(entry_exists!(be, r2_ts));
            assert!(entry_exists!(be, r3_ts));

            assert!(matches!(be.reap_tombstones(&CID_THREE), Ok(2)));

            assert!(!entry_exists!(be, r1_ts));
            assert!(!entry_exists!(be, r2_ts));
            assert!(!entry_exists!(be, r3_ts));

            // Nothing left
            assert!(matches!(be.reap_tombstones(&CID_THREE), Ok(0)));

            assert!(!entry_exists!(be, r1_ts));
            assert!(!entry_exists!(be, r2_ts));
            assert!(!entry_exists!(be, r3_ts));
        });
    }

    #[test]
    fn test_be_backup_restore() {
        let db_backup_file_name = format!(
            "{}/.backup_test.json",
            option_env!("OUT_DIR").unwrap_or("/tmp")
        );
        eprintln!(" ⚠️   {db_backup_file_name}");
        run_test!(|be: &mut BackendWriteTransaction| {
            // Important! Need db metadata setup!
            be.reset_db_s_uuid().unwrap();
            be.reset_db_d_uuid().unwrap();
            be.set_db_ts_max(Duration::from_secs(1)).unwrap();

            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("userid", Value::from("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("userid", Value::from("alice"));
            e2.add_ava("uuid", Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"));

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava("userid", Value::from("lucy"));
            e3.add_ava("uuid", Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"));

            let ve1 = unsafe { e1.clone().into_sealed_new() };
            let ve2 = unsafe { e2.clone().into_sealed_new() };
            let ve3 = unsafe { e3.clone().into_sealed_new() };

            assert!(be.create(&CID_ZERO, vec![ve1, ve2, ve3]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));
            assert!(entry_exists!(be, e3));

            let result = fs::remove_file(&db_backup_file_name);

            match result {
                Err(e) => {
                    // if the error is the file is not found, that's what we want so continue,
                    // otherwise return the error
                    match e.kind() {
                        std::io::ErrorKind::NotFound => {}
                        _ => (),
                    }
                }
                _ => (),
            }

            be.backup(&db_backup_file_name).expect("Backup failed!");
            be.restore(&db_backup_file_name).expect("Restore failed!");

            assert!(be.verify().is_empty());
        });
    }

    #[test]
    fn test_be_backup_restore_tampered() {
        let db_backup_file_name = format!(
            "{}/.backup2_test.json",
            option_env!("OUT_DIR").unwrap_or("/tmp")
        );
        eprintln!(" ⚠️   {db_backup_file_name}");
        run_test!(|be: &mut BackendWriteTransaction| {
            // Important! Need db metadata setup!
            be.reset_db_s_uuid().unwrap();
            be.reset_db_d_uuid().unwrap();
            be.set_db_ts_max(Duration::from_secs(1)).unwrap();
            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("userid", Value::from("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("userid", Value::from("alice"));
            e2.add_ava("uuid", Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"));

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava("userid", Value::from("lucy"));
            e3.add_ava("uuid", Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"));

            let ve1 = unsafe { e1.clone().into_sealed_new() };
            let ve2 = unsafe { e2.clone().into_sealed_new() };
            let ve3 = unsafe { e3.clone().into_sealed_new() };

            assert!(be.create(&CID_ZERO, vec![ve1, ve2, ve3]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));
            assert!(entry_exists!(be, e3));

            let result = fs::remove_file(&db_backup_file_name);

            match result {
                Err(e) => {
                    // if the error is the file is not found, that's what we want so continue,
                    // otherwise return the error
                    match e.kind() {
                        std::io::ErrorKind::NotFound => {}
                        _ => (),
                    }
                }
                _ => (),
            }

            be.backup(&db_backup_file_name).expect("Backup failed!");

            // Now here, we need to tamper with the file.
            let serialized_string = fs::read_to_string(&db_backup_file_name).unwrap();
            let mut dbbak: DbBackup = serde_json::from_str(&serialized_string).unwrap();

            match &mut dbbak {
                DbBackup::V1(_) => {
                    // We no longer use these format versions!
                    unreachable!()
                }
                DbBackup::V2 {
                    db_s_uuid: _,
                    db_d_uuid: _,
                    db_ts_max: _,
                    entries,
                } => {
                    let _ = entries.pop();
                }
            };

            let serialized_entries_str = serde_json::to_string_pretty(&dbbak).unwrap();
            fs::write(&db_backup_file_name, serialized_entries_str).unwrap();

            be.restore(&db_backup_file_name).expect("Restore failed!");

            assert!(be.verify().is_empty());
        });
    }

    #[test]
    fn test_be_sid_generation_and_reset() {
        run_test!(|be: &mut BackendWriteTransaction| {
            let sid1 = be.get_db_s_uuid();
            let sid2 = be.get_db_s_uuid();
            assert!(sid1 == sid2);
            let sid3 = be.reset_db_s_uuid().unwrap();
            assert!(sid1 != sid3);
            let sid4 = be.get_db_s_uuid();
            assert!(sid3 == sid4);
        });
    }

    #[test]
    fn test_be_reindex_empty() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Add some test data?
            let missing = be.missing_idxs().unwrap();
            assert!(missing.len() == 7);
            assert!(be.reindex().is_ok());
            let missing = be.missing_idxs().unwrap();
            debug!("{:?}", missing);
            assert!(missing.is_empty());
        });
    }

    #[test]
    fn test_be_reindex_data() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Add some test data?
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("name", Value::new_iname("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            let e1 = unsafe { e1.into_sealed_new() };

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("name", Value::new_iname("claire"));
            e2.add_ava("uuid", Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f"));
            let e2 = unsafe { e2.into_sealed_new() };

            be.create(&CID_ZERO, vec![e1, e2]).unwrap();

            // purge indexes
            be.purge_idxs().unwrap();
            // Check they are gone
            let missing = be.missing_idxs().unwrap();
            assert!(missing.len() == 7);
            assert!(be.reindex().is_ok());
            let missing = be.missing_idxs().unwrap();
            debug!("{:?}", missing);
            assert!(missing.is_empty());
            // check name and uuid ids on eq, sub, pres

            idl_state!(be, "name", IndexType::Equality, "william", Some(vec![1]));

            idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![2]));

            idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1, 2]));

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(vec![1])
            );

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "bd651620-00dd-426b-aaa0-4494f7b7906f",
                Some(vec![2])
            );

            idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![1, 2]));

            // Show what happens with empty

            idl_state!(
                be,
                "name",
                IndexType::Equality,
                "not-exist",
                Some(Vec::new())
            );

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "fake-0079-4b8c-8a56-593b22aa44d1",
                Some(Vec::new())
            );

            let uuid_p_idl = be
                .load_test_idl(
                    &"not_indexed".to_string(),
                    IndexType::Presence,
                    &"_".to_string(),
                )
                .unwrap(); // unwrap the result
            assert_eq!(uuid_p_idl, None);

            // Check name2uuid
            let claire_uuid = uuid!("bd651620-00dd-426b-aaa0-4494f7b7906f");
            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");

            assert!(be.name2uuid("claire") == Ok(Some(claire_uuid)));
            assert!(be.name2uuid("william") == Ok(Some(william_uuid)));
            assert!(be.name2uuid("db237e8a-0079-4b8c-8a56-593b22aa44d1") == Ok(None));
            // check uuid2spn
            assert!(be.uuid2spn(claire_uuid) == Ok(Some(Value::new_iname("claire"))));
            assert!(be.uuid2spn(william_uuid) == Ok(Some(Value::new_iname("william"))));
            // check uuid2rdn
            assert!(be.uuid2rdn(claire_uuid) == Ok(Some("name=claire".to_string())));
            assert!(be.uuid2rdn(william_uuid) == Ok(Some("name=william".to_string())));
        });
    }

    #[test]
    fn test_be_index_create_delete_simple() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // First, setup our index tables!
            assert!(be.reindex().is_ok());
            // Test that on entry create, the indexes are made correctly.
            // this is a similar case to reindex.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("name", Value::from("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            let e1 = unsafe { e1.into_sealed_new() };

            let rset = be.create(&CID_ZERO, vec![e1]).unwrap();
            let mut rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            let e1 = rset.pop().unwrap();

            idl_state!(be, "name", IndexType::Equality, "william", Some(vec![1]));

            idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1]));

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(vec![1])
            );

            idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![1]));

            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            assert!(be.name2uuid("william") == Ok(Some(william_uuid)));
            assert!(be.uuid2spn(william_uuid) == Ok(Some(Value::from("william"))));
            assert!(be.uuid2rdn(william_uuid) == Ok(Some("name=william".to_string())));

            // == Now we reap_tombstones, and assert we removed the items.
            let e1_ts = unsafe { e1.to_tombstone(CID_ONE.clone()).into_sealed_committed() };
            assert!(be.modify(&CID_ONE, &[e1], &[e1_ts]).is_ok());
            be.reap_tombstones(&CID_TWO).unwrap();

            idl_state!(be, "name", IndexType::Equality, "william", Some(Vec::new()));

            idl_state!(be, "name", IndexType::Presence, "_", Some(Vec::new()));

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(Vec::new())
            );

            idl_state!(be, "uuid", IndexType::Presence, "_", Some(Vec::new()));

            assert!(be.name2uuid("william") == Ok(None));
            assert!(be.uuid2spn(william_uuid) == Ok(None));
            assert!(be.uuid2rdn(william_uuid) == Ok(None));
        })
    }

    #[test]
    fn test_be_index_create_delete_multi() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // delete multiple entries at a time, without deleting others
            // First, setup our index tables!
            assert!(be.reindex().is_ok());
            // Test that on entry create, the indexes are made correctly.
            // this is a similar case to reindex.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("name", Value::new_iname("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            let e1 = unsafe { e1.into_sealed_new() };

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("name", Value::new_iname("claire"));
            e2.add_ava("uuid", Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f"));
            let e2 = unsafe { e2.into_sealed_new() };

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava("userid", Value::new_iname("lucy"));
            e3.add_ava("uuid", Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"));
            let e3 = unsafe { e3.into_sealed_new() };

            let mut rset = be.create(&CID_ZERO, vec![e1, e2, e3]).unwrap();
            rset.remove(1);
            let mut rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            let e1 = rset.pop().unwrap();
            let e3 = rset.pop().unwrap();

            // Now remove e1, e3.
            let e1_ts = unsafe { e1.to_tombstone(CID_ONE.clone()).into_sealed_committed() };
            let e3_ts = unsafe { e3.to_tombstone(CID_ONE.clone()).into_sealed_committed() };
            assert!(be.modify(&CID_ONE, &[e1, e3], &[e1_ts, e3_ts]).is_ok());
            be.reap_tombstones(&CID_TWO).unwrap();

            idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![2]));

            idl_state!(be, "name", IndexType::Presence, "_", Some(vec![2]));

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "bd651620-00dd-426b-aaa0-4494f7b7906f",
                Some(vec![2])
            );

            idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![2]));

            let claire_uuid = uuid!("bd651620-00dd-426b-aaa0-4494f7b7906f");
            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            let lucy_uuid = uuid!("7b23c99d-c06b-4a9a-a958-3afa56383e1d");

            assert!(be.name2uuid("claire") == Ok(Some(claire_uuid)));
            let x = be.uuid2spn(claire_uuid);
            trace!(?x);
            assert!(be.uuid2spn(claire_uuid) == Ok(Some(Value::new_iname("claire"))));
            assert!(be.uuid2rdn(claire_uuid) == Ok(Some("name=claire".to_string())));

            assert!(be.name2uuid("william") == Ok(None));
            assert!(be.uuid2spn(william_uuid) == Ok(None));
            assert!(be.uuid2rdn(william_uuid) == Ok(None));

            assert!(be.name2uuid("lucy") == Ok(None));
            assert!(be.uuid2spn(lucy_uuid) == Ok(None));
            assert!(be.uuid2rdn(lucy_uuid) == Ok(None));
        })
    }

    #[test]
    fn test_be_index_modify_simple() {
        run_test!(|be: &mut BackendWriteTransaction| {
            assert!(be.reindex().is_ok());
            // modify with one type, ensuring we clean the indexes behind
            // us. For the test to be "accurate" we must add one attr, remove one attr
            // and change one attr.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("name", Value::new_iname("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            e1.add_ava("ta", Value::from("test"));
            let e1 = unsafe { e1.into_sealed_new() };

            let rset = be.create(&CID_ZERO, vec![e1]).unwrap();
            let rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            // Now, alter the new entry.
            let mut ce1 = unsafe { rset[0].as_ref().clone().into_invalid() };
            // add something.
            ce1.add_ava("tb", Value::from("test"));
            // remove something.
            ce1.purge_ava("ta");
            // mod something.
            ce1.purge_ava("name");
            ce1.add_ava("name", Value::new_iname("claire"));

            let ce1 = unsafe { ce1.into_sealed_committed() };

            be.modify(&CID_ZERO, &rset, &[ce1]).unwrap();

            // Now check the idls
            idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![1]));

            idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1]));

            idl_state!(be, "tb", IndexType::Equality, "test", Some(vec![1]));

            idl_state!(be, "ta", IndexType::Equality, "test", Some(vec![]));

            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            assert!(be.name2uuid("william") == Ok(None));
            assert!(be.name2uuid("claire") == Ok(Some(william_uuid)));
            assert!(be.uuid2spn(william_uuid) == Ok(Some(Value::new_iname("claire"))));
            assert!(be.uuid2rdn(william_uuid) == Ok(Some("name=claire".to_string())));
        })
    }

    #[test]
    fn test_be_index_modify_rename() {
        run_test!(|be: &mut BackendWriteTransaction| {
            assert!(be.reindex().is_ok());
            // test when we change name AND uuid
            // This will be needing to be correct for conflicts when we add
            // replication support!
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("name", Value::new_iname("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            let e1 = unsafe { e1.into_sealed_new() };

            let rset = be.create(&CID_ZERO, vec![e1]).unwrap();
            let rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            // Now, alter the new entry.
            let mut ce1 = unsafe { rset[0].as_ref().clone().into_invalid() };
            ce1.purge_ava("name");
            ce1.purge_ava("uuid");
            ce1.add_ava("name", Value::new_iname("claire"));
            ce1.add_ava("uuid", Value::from("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8"));
            let ce1 = unsafe { ce1.into_sealed_committed() };

            be.modify(&CID_ZERO, &rset, &[ce1]).unwrap();

            idl_state!(be, "name", IndexType::Equality, "claire", Some(vec![1]));

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "04091a7a-6ce4-42d2-abf5-c2ce244ac9e8",
                Some(vec![1])
            );

            idl_state!(be, "name", IndexType::Presence, "_", Some(vec![1]));
            idl_state!(be, "uuid", IndexType::Presence, "_", Some(vec![1]));

            idl_state!(
                be,
                "uuid",
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(Vec::new())
            );
            idl_state!(be, "name", IndexType::Equality, "william", Some(Vec::new()));

            let claire_uuid = uuid!("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8");
            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            assert!(be.name2uuid("william") == Ok(None));
            assert!(be.name2uuid("claire") == Ok(Some(claire_uuid)));
            assert!(be.uuid2spn(william_uuid) == Ok(None));
            assert!(be.uuid2rdn(william_uuid) == Ok(None));
            assert!(be.uuid2spn(claire_uuid) == Ok(Some(Value::new_iname("claire"))));
            assert!(be.uuid2rdn(claire_uuid) == Ok(Some("name=claire".to_string())));
        })
    }

    #[test]
    fn test_be_index_search_simple() {
        run_test!(|be: &mut BackendWriteTransaction| {
            assert!(be.reindex().is_ok());

            // Create a test entry with some indexed / unindexed values.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("name", Value::new_iname("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            e1.add_ava("no-index", Value::from("william"));
            e1.add_ava("other-no-index", Value::from("william"));
            let e1 = unsafe { e1.into_sealed_new() };

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("name", Value::new_iname("claire"));
            e2.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d2"));
            let e2 = unsafe { e2.into_sealed_new() };

            let _rset = be.create(&CID_ZERO, vec![e1, e2]).unwrap();
            // Test fully unindexed
            let f_un =
                unsafe { filter_resolved!(f_eq("no-index", PartialValue::new_utf8s("william"))) };

            let (r, _plan) = be.filter2idl(f_un.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            // Test that a fully indexed search works
            let feq = unsafe { filter_resolved!(f_eq("name", PartialValue::new_utf8s("william"))) };

            let (r, _plan) = be.filter2idl(feq.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }

            // Test and/or
            //   full index and
            let f_in_and = unsafe {
                filter_resolved!(f_and!([
                    f_eq("name", PartialValue::new_utf8s("william")),
                    f_eq(
                        "uuid",
                        PartialValue::new_utf8s("db237e8a-0079-4b8c-8a56-593b22aa44d1")
                    )
                ]))
            };

            let (r, _plan) = be.filter2idl(f_in_and.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }

            //   partial index and
            let f_p1 = unsafe {
                filter_resolved!(f_and!([
                    f_eq("name", PartialValue::new_utf8s("william")),
                    f_eq("no-index", PartialValue::new_utf8s("william"))
                ]))
            };

            let f_p2 = unsafe {
                filter_resolved!(f_and!([
                    f_eq("name", PartialValue::new_utf8s("william")),
                    f_eq("no-index", PartialValue::new_utf8s("william"))
                ]))
            };

            let (r, _plan) = be.filter2idl(f_p1.to_inner(), 0).unwrap();
            match r {
                IdList::Partial(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }

            let (r, _plan) = be.filter2idl(f_p2.to_inner(), 0).unwrap();
            match r {
                IdList::Partial(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }

            //   no index and
            let f_no_and = unsafe {
                filter_resolved!(f_and!([
                    f_eq("no-index", PartialValue::new_utf8s("william")),
                    f_eq("other-no-index", PartialValue::new_utf8s("william"))
                ]))
            };

            let (r, _plan) = be.filter2idl(f_no_and.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            //   full index or
            let f_in_or = unsafe {
                filter_resolved!(f_or!([f_eq("name", PartialValue::new_utf8s("william"))]))
            };

            let (r, _plan) = be.filter2idl(f_in_or.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }
            //   partial (aka allids) or
            let f_un_or = unsafe {
                filter_resolved!(f_or!([f_eq(
                    "no-index",
                    PartialValue::new_utf8s("william")
                )]))
            };

            let (r, _plan) = be.filter2idl(f_un_or.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            // Test root andnot
            let f_r_andnot = unsafe {
                filter_resolved!(f_andnot(f_eq("name", PartialValue::new_utf8s("william"))))
            };

            let (r, _plan) = be.filter2idl(f_r_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(Vec::new()));
                }
                _ => {
                    panic!("");
                }
            }

            // test andnot as only in and
            let f_and_andnot = unsafe {
                filter_resolved!(f_and!([f_andnot(f_eq(
                    "name",
                    PartialValue::new_utf8s("william")
                ))]))
            };

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(Vec::new()));
                }
                _ => {
                    panic!("");
                }
            }
            // test andnot as only in or
            let f_or_andnot = unsafe {
                filter_resolved!(f_or!([f_andnot(f_eq(
                    "name",
                    PartialValue::new_utf8s("william")
                ))]))
            };

            let (r, _plan) = be.filter2idl(f_or_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(Vec::new()));
                }
                _ => {
                    panic!("");
                }
            }

            // test andnot in and (first) with name
            let f_and_andnot = unsafe {
                filter_resolved!(f_and!([
                    f_andnot(f_eq("name", PartialValue::new_utf8s("claire"))),
                    f_pres("name")
                ]))
            };

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    debug!("{:?}", idl);
                    assert!(idl == IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }
            // test andnot in and (last) with name
            let f_and_andnot = unsafe {
                filter_resolved!(f_and!([
                    f_pres("name"),
                    f_andnot(f_eq("name", PartialValue::new_utf8s("claire")))
                ]))
            };

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }
            // test andnot in and (first) with no-index
            let f_and_andnot = unsafe {
                filter_resolved!(f_and!([
                    f_andnot(f_eq("name", PartialValue::new_utf8s("claire"))),
                    f_pres("no-index")
                ]))
            };

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }
            // test andnot in and (last) with no-index
            let f_and_andnot = unsafe {
                filter_resolved!(f_and!([
                    f_pres("no-index"),
                    f_andnot(f_eq("name", PartialValue::new_utf8s("claire")))
                ]))
            };

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            //   empty or
            let f_e_or = unsafe { filter_resolved!(f_or!([])) };

            let (r, _plan) = be.filter2idl(f_e_or.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![]));
                }
                _ => {
                    panic!("");
                }
            }

            let f_e_and = unsafe { filter_resolved!(f_and!([])) };

            let (r, _plan) = be.filter2idl(f_e_and.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert!(idl == IDLBitRange::from_iter(vec![]));
                }
                _ => {
                    panic!("");
                }
            }
        })
    }

    #[test]
    fn test_be_index_search_missing() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Test where the index is in schema but not created (purge idxs)
            // should fall back to an empty set because we can't satisfy the term
            be.purge_idxs().unwrap();
            debug!("{:?}", be.missing_idxs().unwrap());
            let f_eq =
                unsafe { filter_resolved!(f_eq("name", PartialValue::new_utf8s("william"))) };

            let (r, _plan) = be.filter2idl(f_eq.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }
        })
    }

    #[test]
    fn test_be_index_slope_generation() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Create some test entry with some indexed / unindexed values.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava("name", Value::new_iname("william"));
            e1.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            e1.add_ava("ta", Value::from("dupe"));
            e1.add_ava("tb", Value::from("1"));
            let e1 = unsafe { e1.into_sealed_new() };

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava("name", Value::new_iname("claire"));
            e2.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d2"));
            e2.add_ava("ta", Value::from("dupe"));
            e2.add_ava("tb", Value::from("1"));
            let e2 = unsafe { e2.into_sealed_new() };

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava("name", Value::new_iname("benny"));
            e3.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d3"));
            e3.add_ava("ta", Value::from("dupe"));
            e3.add_ava("tb", Value::from("2"));
            let e3 = unsafe { e3.into_sealed_new() };

            let _rset = be.create(&CID_ZERO, vec![e1, e2, e3]).unwrap();

            // If the slopes haven't been generated yet, there are some hardcoded values
            // that we can use instead. They aren't generated until a first re-index.
            assert!(!be.is_idx_slopeyness_generated().unwrap());

            let ta_eq_slope = be
                .get_idx_slope(&IdxKey::new("ta", IndexType::Equality))
                .unwrap();
            assert_eq!(ta_eq_slope, 45);

            let tb_eq_slope = be
                .get_idx_slope(&IdxKey::new("tb", IndexType::Equality))
                .unwrap();
            assert_eq!(tb_eq_slope, 45);

            let name_eq_slope = be
                .get_idx_slope(&IdxKey::new("name", IndexType::Equality))
                .unwrap();
            assert_eq!(name_eq_slope, 1);
            let uuid_eq_slope = be
                .get_idx_slope(&IdxKey::new("uuid", IndexType::Equality))
                .unwrap();
            assert_eq!(uuid_eq_slope, 1);

            let name_pres_slope = be
                .get_idx_slope(&IdxKey::new("name", IndexType::Presence))
                .unwrap();
            assert_eq!(name_pres_slope, 90);
            let uuid_pres_slope = be
                .get_idx_slope(&IdxKey::new("uuid", IndexType::Presence))
                .unwrap();
            assert_eq!(uuid_pres_slope, 90);
            // Check the slopes are what we expect for hardcoded values.

            // Now check slope generation for the values. Today these are calculated
            // at reindex time, so we now perform the re-index.
            assert!(be.reindex().is_ok());
            assert!(be.is_idx_slopeyness_generated().unwrap());

            let ta_eq_slope = be
                .get_idx_slope(&IdxKey::new("ta", IndexType::Equality))
                .unwrap();
            assert_eq!(ta_eq_slope, 200);

            let tb_eq_slope = be
                .get_idx_slope(&IdxKey::new("tb", IndexType::Equality))
                .unwrap();
            assert_eq!(tb_eq_slope, 133);

            let name_eq_slope = be
                .get_idx_slope(&IdxKey::new("name", IndexType::Equality))
                .unwrap();
            assert_eq!(name_eq_slope, 51);
            let uuid_eq_slope = be
                .get_idx_slope(&IdxKey::new("uuid", IndexType::Equality))
                .unwrap();
            assert_eq!(uuid_eq_slope, 51);

            let name_pres_slope = be
                .get_idx_slope(&IdxKey::new("name", IndexType::Presence))
                .unwrap();
            assert_eq!(name_pres_slope, 200);
            let uuid_pres_slope = be
                .get_idx_slope(&IdxKey::new("uuid", IndexType::Presence))
                .unwrap();
            assert_eq!(uuid_pres_slope, 200);
        })
    }

    #[test]
    fn test_be_limits_allids() {
        run_test!(|be: &mut BackendWriteTransaction| {
            let mut lim_allow_allids = Limits::unlimited();
            lim_allow_allids.unindexed_allow = true;

            let mut lim_deny_allids = Limits::unlimited();
            lim_deny_allids.unindexed_allow = false;

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava("userid", Value::from("william"));
            e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            e.add_ava("nonexist", Value::from("x"));
            let e = unsafe { e.into_sealed_new() };
            let single_result = be.create(&CID_ZERO, vec![e.clone()]);

            assert!(single_result.is_ok());
            let filt = unsafe {
                e.filter_from_attrs(&[AttrString::from("nonexist")])
                    .expect("failed to generate filter")
                    .into_valid_resolved()
            };
            // check allow on allids
            let res = be.search(&lim_allow_allids, &filt);
            assert!(res.is_ok());
            let res = be.exists(&lim_allow_allids, &filt);
            assert!(res.is_ok());

            // check deny on allids
            let res = be.search(&lim_deny_allids, &filt);
            assert!(res == Err(OperationError::ResourceLimit));
            let res = be.exists(&lim_deny_allids, &filt);
            assert!(res == Err(OperationError::ResourceLimit));
        })
    }

    #[test]
    fn test_be_limits_results_max() {
        run_test!(|be: &mut BackendWriteTransaction| {
            let mut lim_allow = Limits::unlimited();
            lim_allow.search_max_results = usize::MAX;

            let mut lim_deny = Limits::unlimited();
            lim_deny.search_max_results = 0;

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava("userid", Value::from("william"));
            e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            e.add_ava("nonexist", Value::from("x"));
            let e = unsafe { e.into_sealed_new() };
            let single_result = be.create(&CID_ZERO, vec![e.clone()]);
            assert!(single_result.is_ok());

            let filt = unsafe {
                e.filter_from_attrs(&[AttrString::from("nonexist")])
                    .expect("failed to generate filter")
                    .into_valid_resolved()
            };

            // --> This is the all ids path (unindexed)
            // check allow on entry max
            let res = be.search(&lim_allow, &filt);
            assert!(res.is_ok());
            let res = be.exists(&lim_allow, &filt);
            assert!(res.is_ok());

            // check deny on entry max
            let res = be.search(&lim_deny, &filt);
            assert!(res == Err(OperationError::ResourceLimit));
            // we don't limit on exists because we never load the entries.
            let res = be.exists(&lim_deny, &filt);
            assert!(res.is_ok());

            // --> This will shortcut due to indexing.
            assert!(be.reindex().is_ok());
            let res = be.search(&lim_deny, &filt);
            assert!(res == Err(OperationError::ResourceLimit));
            // we don't limit on exists because we never load the entries.
            let res = be.exists(&lim_deny, &filt);
            assert!(res.is_ok());
        })
    }

    #[test]
    fn test_be_limits_partial_filter() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // This relies on how we do partials, so it could be a bit sensitive.
            // A partial is generated after an allids + indexed in a single and
            // as we require both conditions to exist. Allids comes from unindexed
            // terms. we need to ensure we don't hit partial threshold too.
            //
            // This means we need an and query where the first term is allids
            // and the second is indexed, but without the filter shortcutting.
            //
            // To achieve this we need a monstrously evil query.
            //
            let mut lim_allow = Limits::unlimited();
            lim_allow.search_max_filter_test = usize::MAX;

            let mut lim_deny = Limits::unlimited();
            lim_deny.search_max_filter_test = 0;

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava("name", Value::new_iname("william"));
            e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
            e.add_ava("nonexist", Value::from("x"));
            e.add_ava("nonexist", Value::from("y"));
            let e = unsafe { e.into_sealed_new() };
            let single_result = be.create(&CID_ZERO, vec![e]);
            assert!(single_result.is_ok());

            // Reindex so we have things in place for our query
            assert!(be.reindex().is_ok());

            // 🚨 This is evil!
            // The and allows us to hit "allids + indexed -> partial".
            // the or terms prevent re-arrangement. They can't be folded or dead
            // term elimed either.
            //
            // This means the f_or nonexist will become allids and the second will be indexed
            // due to f_eq userid in both with the result of william.
            //
            // This creates a partial, and because it's the first iteration in the loop, this
            // doesn't encounter partial threshold testing.
            let filt = unsafe {
                filter_resolved!(f_and!([
                    f_or!([
                        f_eq("nonexist", PartialValue::new_utf8s("x")),
                        f_eq("nonexist", PartialValue::new_utf8s("y"))
                    ]),
                    f_or!([
                        f_eq("name", PartialValue::new_utf8s("claire")),
                        f_eq("name", PartialValue::new_utf8s("william"))
                    ]),
                ]))
            };

            let res = be.search(&lim_allow, &filt);
            assert!(res.is_ok());
            let res = be.exists(&lim_allow, &filt);
            assert!(res.is_ok());

            // check deny on entry max
            let res = be.search(&lim_deny, &filt);
            assert!(res == Err(OperationError::ResourceLimit));
            // we don't limit on exists because we never load the entries.
            let res = be.exists(&lim_deny, &filt);
            assert!(res == Err(OperationError::ResourceLimit));
        })
    }

    #[test]
    fn test_be_mulitple_create() {
        sketching::test_init();

        // This is a demo idxmeta, purely for testing.
        let idxmeta = vec![IdxKey {
            attr: AttrString::from("uuid"),
            itype: IndexType::Equality,
        }];

        let be_a = Backend::new(BackendConfig::new_test("main"), idxmeta.clone(), false)
            .expect("Failed to setup backend");

        let be_b = Backend::new(BackendConfig::new_test("db_2"), idxmeta, false)
            .expect("Failed to setup backend");

        let mut be_a_txn = be_a.write();
        let mut be_b_txn = be_b.write();

        assert!(be_a_txn.get_db_s_uuid() != be_b_txn.get_db_s_uuid());

        // Create into A
        let mut e: Entry<EntryInit, EntryNew> = Entry::new();
        e.add_ava("userid", Value::from("william"));
        e.add_ava("uuid", Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"));
        let e = unsafe { e.into_sealed_new() };

        let single_result = be_a_txn.create(&CID_ZERO, vec![e]);

        assert!(single_result.is_ok());

        // Assert it's in A but not B.
        let filt = unsafe { filter_resolved!(f_eq("userid", PartialValue::new_utf8s("william"))) };

        let lims = Limits::unlimited();

        let r = be_a_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").len() == 1);

        let r = be_b_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").is_empty());

        // Create into B
        let mut e: Entry<EntryInit, EntryNew> = Entry::new();
        e.add_ava("userid", Value::from("claire"));
        e.add_ava("uuid", Value::from("0c680959-0944-47d6-9dea-53304d124266"));
        let e = unsafe { e.into_sealed_new() };

        let single_result = be_b_txn.create(&CID_ZERO, vec![e]);

        assert!(single_result.is_ok());

        // Assert it's in B but not A
        let filt = unsafe { filter_resolved!(f_eq("userid", PartialValue::new_utf8s("claire"))) };

        let lims = Limits::unlimited();

        let r = be_a_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").is_empty());

        let r = be_b_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").len() == 1);
    }
}