Yak hassling (#2059)

* trying this query thing again
* if error show error not panic
* clippyism
* moving dependencies around and fixing log messages for healthcheck
* cleaning up some comment mess
* fixing the "debug thing breaks packaging" issue and test failures

.gitignore

@@ -16,6 +16,8 @@ vendor.tar.*
 tools/orca/example_profiles/small/orca-edited.toml
 /docs/
 .vscode/
+# webui things we don't need
+*.d.ts
 
 # kanidm simple packaging
 deployment-config/

server/core/src/https/v1.rs

@@ -1425,7 +1425,6 @@ pub async fn auth_valid(
     to_axum_response(res)
 }
 
-#[cfg(debug_assertions)]
 pub async fn debug_ipinfo(
     State(_state): State<ServerState>,
     TrustedClientIp(ip_addr): TrustedClientIp,
@@ -1435,7 +1434,7 @@ pub async fn debug_ipinfo(
 
 #[instrument(skip(state))]
 pub fn router(state: ServerState) -> Router<ServerState> {
-    let mut router = Router::new()
+    let router = Router::new()
         .route("/v1/oauth2", get(super::oauth2::oauth2_get))
         .route("/v1/oauth2/_basic", post(super::oauth2::oauth2_basic_post))
         .route(
@@ -1737,8 +1736,14 @@ pub fn router(state: ServerState) -> Router<ServerState> {
         )
         .with_state(state)
         .layer(from_fn(dont_cache_me));
-    if cfg!(debug_assertions) {
+    if cfg!(debug_assertions) || cfg!(test) {
-        router = router.route("/v1/debug/ipinfo", get(debug_ipinfo));
+        add_debug_info(router)
+    } else {
+        router
     }
-    router
+}
+
+// #[cfg(any(debug_assertions, test))]
+fn add_debug_info(router: Router<ServerState>) -> Router<ServerState> {
+    router.route("/v1/debug/ipinfo", get(debug_ipinfo))
 }

server/daemon/src/main.rs

@@ -638,7 +638,7 @@ async fn main() -> ExitCode {
                     client = match &sconfig.tls_chain {
                         None => client,
                         Some(ca_cert) => {
-                            debug!("Trying to load {}", ca_cert);
+                            debug!("Trying to load {} to build a CA cert path", ca_cert);
                             // if the ca_cert file exists, then we'll use it
                             let ca_cert_path = PathBuf::from(ca_cert);
                             match ca_cert_path.exists() {
@@ -666,7 +666,7 @@ async fn main() -> ExitCode {
                                     let ca_cert_parsed = match reqwest::Certificate::from_pem(content.as_bytes()) {
                                         Ok(val) => val,
                                         Err(e) =>{
-                                            error!("Failed to parse {} as a valid certificate!\nError: {:?}", ca_cert, e);
+                                            error!("Failed to parse {} into CA certificate!\nError: {:?}", ca_cert, e);
                                         return ExitCode::FAILURE
                                         }
                                     };

server/lib/Cargo.toml

@@ -50,6 +50,8 @@ regex = { workspace = true, features = [
     "unicode",
     "unicode-gencat",
 ] }
+rusqlite = { workspace = true, features = ["array", "bundled"] }
+
 serde = { workspace = true, features = ["derive"] }
 serde_cbor = { workspace = true }
 serde_json = { workspace = true }
@@ -80,11 +82,9 @@ serde_with = { workspace = true }
 
 # because windows really can't build without the bundled one
 [target.'cfg(target_family = "windows")'.dependencies]
-rusqlite = { workspace = true, features = ["bundled"] }
 whoami = { workspace = true }
 
 [target.'cfg(not(target_family = "windows"))'.dependencies]
-rusqlite = { workspace = true }
 kanidm_utils_users = { workspace = true }
 
 [features]

server/lib/src/be/idl_arc_sqlite.rs

@@ -565,7 +565,7 @@ impl<'a> IdlArcSqliteTransaction for IdlArcSqliteWriteTransaction<'a> {
 }
 
 impl<'a> IdlArcSqliteWriteTransaction<'a> {
-    #[cfg(debug_assertions)]
+    #[cfg(any(test, debug_assertions))]
     #[instrument(level = "debug", name = "idl_arc_sqlite::clear_cache", skip_all)]
     pub fn clear_cache(&mut self) -> Result<(), OperationError> {
         // I'm not sure rn if I want to reload these? If we reload these we kind of

server/lib/src/be/idl_sqlite.rs

@@ -8,6 +8,7 @@ use std::time::Duration;
 use hashbrown::HashMap;
 use idlset::v2::IDLBitRange;
 use kanidm_proto::v1::{ConsistencyError, OperationError};
+use rusqlite::vtab::array::Array;
 use rusqlite::{Connection, OpenFlags, OptionalExtension};
 use uuid::Uuid;
 
@@ -160,44 +161,53 @@ pub trait IdlSqliteTransaction {
                 let mut stmt = self
                     .get_conn()?
                     .prepare(&format!(
-                        "SELECT id, data FROM {}.id2entry WHERE id = :idl",
+                        "SELECT id, data FROM {}.id2entry
+                         WHERE id IN rarray(:idli)",
                         self.get_db_name()
                     ))
                     .map_err(sqlite_error)?;
 
-                // TODO #258: Can this actually just load in a single select?
+                // turn them into i64's
-                // TODO #258: I have no idea how to make this an iterator chain ... so what
+                let mut id_list: Vec<i64> = vec![];
-                // I have now is probably really bad :(
-                let mut results = Vec::new();
-
-                // Faster to iterate over the compressed form believe it or not.
-                /*
-                let decompressed: Result<Vec<i64>, _> = idli.into_iter()
-                    .map(|u| i64::try_from(u).map_err(|_| OperationError::InvalidEntryId))
-                    .collect();
-                */
-
                 for id in idli {
-                    let iid = i64::try_from(id).map_err(|_| OperationError::InvalidEntryId)?;
+                    id_list.push(i64::try_from(id).map_err(|_| OperationError::InvalidEntryId)?);
-                    let id2entry_iter = stmt
+                }
-                        .query_map([&iid], |row| {
+                // turn them into rusqlite values
-                            Ok(IdSqliteEntry {
+                let id_list: Array = std::rc::Rc::new(
-                                id: row.get(0)?,
+                    id_list
-                                data: row.get(1)?,
+                        .into_iter()
-                            })
+                        .map(rusqlite::types::Value::from)
-                        })
+                        .collect::<Vec<rusqlite::types::Value>>(),
-                        .map_err(sqlite_error)?;
+                );
 
-                    let r: Result<Vec<_>, _> = id2entry_iter
+                let mut results: Vec<IdRawEntry> = vec![];
-                        .map(|v| {
+
-                            v.map_err(sqlite_error).and_then(|ise| {
+                let rows = stmt.query_map(named_params! {":idli": &id_list}, |row| {
-                                // Convert the idsqlite to id raw
+                    Ok(IdSqliteEntry {
-                                ise.try_into()
+                        id: row.get(0)?,
-                            })
+                        data: row.get(1)?,
-                        })
+                    })
-                        .collect();
+                });
-                    let mut r = r?;
+                let rows = match rows {
-                    results.append(&mut r);
+                    Ok(rows) => rows,
+                    Err(e) => {
+                        error!("query failed in get_identry_raw: {:?}", e);
+                        return Err(OperationError::SqliteError);
+                    }
+                };
+
+                for row in rows {
+                    match row {
+                        Ok(ise) => {
+                            // Convert the idsqlite to id raw
+                            results.push(ise.try_into()?);
+                        }
+                        // TODO: make this a better error
+                        Err(e) => {
+                            admin_error!(?e, "SQLite Error in get_identry_raw");
+                            return Err(OperationError::SqliteError);
+                        }
+                    }
                 }
                 Ok(results)
             }
@@ -1695,7 +1705,27 @@ impl IdlSqlite {
         let pool = (0..cfg.pool_size)
             .map(|i| {
                 trace!("Opening Connection {}", i);
-                Connection::open_with_flags(cfg.path.as_str(), flags).map_err(sqlite_error)
+                let conn =
+                    Connection::open_with_flags(cfg.path.as_str(), flags).map_err(sqlite_error);
+                match conn {
+                    Ok(conn) => {
+                        // load the rusqlite vtab module to allow for virtual tables
+                        rusqlite::vtab::array::load_module(&conn).map_err(|e| {
+                            admin_error!(
+                                "Failed to load rarray virtual module for sqlite, cannot start! {:?}", e
+                            );
+                            sqlite_error(e)
+                        })?;
+                        Ok(conn)
+                    }
+                    Err(err) => {
+                        admin_error!(
+                            "Failed to start database connection, cannot start! {:?}",
+                            err
+                        );
+                        Err(err)
+                    }
+                }
             })
             .collect::<Result<VecDeque<Connection>, OperationError>>()
             .map_err(|e| {

server/lib/src/be/mod.rs

@@ -1792,7 +1792,7 @@ impl<'a> BackendWriteTransaction<'a> {
         self.set_db_index_version(0)
     }
 
-    #[cfg(debug_assertions)]
+    #[cfg(any(test, debug_assertions))]
     pub fn clear_cache(&mut self) -> Result<(), OperationError> {
         self.get_idlayer().clear_cache()
     }

server/lib/src/server/mod.rs

@@ -1220,7 +1220,7 @@ impl QueryServer {
         }
     }
 
-    #[cfg(debug_assertions)]
+    #[cfg(any(test, debug_assertions))]
     pub async fn clear_cache(&self) -> Result<(), OperationError> {
         let ct = duration_from_epoch_now();
         let mut w_txn = self.write(ct).await;
@@ -1608,7 +1608,7 @@ impl<'a> QueryServerWriteTransaction<'a> {
         Ok(())
     }
 
-    #[cfg(debug_assertions)]
+    #[cfg(any(test, debug_assertions))]
     #[instrument(level = "debug", skip_all)]
     pub fn clear_cache(&mut self) -> Result<(), OperationError> {
         self.be_txn.clear_cache()

server/web_ui/build_wasm.sh

@@ -29,7 +29,7 @@ fi
 
 # we can disable this since we want it to expand
 # shellcheck disable=SC2086
-wasm-pack build ${BUILD_FLAGS} --target web --mode no-install --no-pack || exit 1
+wasm-pack build ${BUILD_FLAGS} --target web --mode no-install --no-pack
 
 touch ./pkg/ANYTHING_HERE_WILL_BE_DELETED_ADD_TO_SRC && \
     rsync --delete-after -r --copy-links -v ./static/* ./pkg/ && \
@@ -37,6 +37,8 @@ touch ./pkg/ANYTHING_HERE_WILL_BE_DELETED_ADD_TO_SRC && \
     cp ../../LICENSE.md ./pkg/
     rm ./pkg/.gitignore
 
-# updates the brotli-compressed files
+if [ -z "${SKIP_BROTLI}" ]; then
-echo "brotli-compressing the WASM file..."
+    # updates the brotli-compressed files
-find ./pkg -name '*.wasm' -exec ./find_best_brotli.sh "{}" \; || exit 1
+    echo "brotli-compressing the WASM file..."
+    find ./pkg -name '*.wasm' -exec ./find_best_brotli.sh "{}" \; || exit 1
+fi

server/web_ui/src/views/mod.rs

@@ -1,3 +1,4 @@
+#![allow(non_camel_case_types)]
 use gloo::console;
 use kanidm_proto::v1::{UiHint, UserAuthToken};
 use serde::{Deserialize, Serialize};