From 63deda350cd98e32ffb0a9802ff6d41e7c13914b Mon Sep 17 00:00:00 2001 From: Firstyear <william@blackhats.net.au> Date: Tue, 4 Mar 2025 10:36:53 +1000 Subject: [PATCH] 20250225 improve test performance (#3459) * Ignore tests that are no longer used. Each time a library or binary is added, that requires compilation to create the *empty* test harness, which then is executed and takes multiple seconds to start up, do nothing, and return success. This removes test's for libraries that aren't actually using or running any tests. Additionally, each time a new test binary is added, that adds a ton of compilation time, but also test execution time as the binary for each test runner must start up, execute, and shutdown. So this merges all the testkit integration tests to a single running which significantly speeds up test execution. * Improve IDL exists behaviour, improve memberof verification Again to improve test performance. This improves the validation of idx existance to be a faster SQLite call, caches the results as needed. Memberof was taking up a large amount of time in verify phases of test finalisation, and so a better in memory version has been added. * Disable TLS native roots when not needed * Cleanup tests that are hitting native certs, or do nothing at all --- Cargo.lock | 1 - Cargo.toml | 1 + libs/client/src/lib.rs | 192 +++++++++++------- server/core/src/lib.rs | 4 +- server/daemon/Cargo.toml | 2 +- server/lib-macros/Cargo.toml | 2 +- server/lib/src/be/idl_arc_sqlite.rs | 92 ++++++++- server/lib/src/be/idl_sqlite.rs | 5 +- server/lib/src/be/idxkey.rs | 6 + server/lib/src/idm/server.rs | 3 +- server/lib/src/plugins/memberof.rs | 79 ++++--- server/lib/src/testkit.rs | 2 +- server/testkit-macros/Cargo.toml | 2 +- server/testkit/Cargo.toml | 4 +- .../oauth2_device_flow.rs | 0 server/testkit/src/lib.rs | 3 +- .../testkit/tests/DO_NOT_ADD_MORE_TESTS_HERE | 11 + server/testkit/tests/integration_test.rs | 3 + server/testkit/tests/self.rs | 57 ------ server/testkit/tests/{ => testkit}/apidocs.rs | 17 +- server/testkit/tests/{ => testkit}/domain.rs | 0 server/testkit/tests/{ => testkit}/group.rs | 4 +- .../tests/{ => testkit}/http_manifest.rs | 13 +- .../tests/{ => testkit}/https_extractors.rs | 52 ++--- .../tests/{ => testkit}/https_middleware.rs | 22 +- .../identity_verification_tests.rs | 10 +- .../tests/{ => testkit}/integration.rs | 0 server/testkit/tests/testkit/mod.rs | 16 ++ .../testkit/tests/{ => testkit}/mtls_test.rs | 0 .../tests/{ => testkit}/oauth2_test.rs | 43 ++-- server/testkit/tests/{ => testkit}/person.rs | 4 +- .../tests/{ => testkit}/proto_v1_test.rs | 8 +- .../testkit/tests/{ => testkit}/scim_test.rs | 59 ------ .../tests/{ => testkit}/service_account.rs | 5 +- server/testkit/tests/{ => testkit}/system.rs | 5 +- server/testkit/tests/{ => testkit}/unix.rs | 0 tools/cli/Cargo.toml | 4 +- tools/device_flow/Cargo.toml | 2 +- tools/iam_migrations/freeipa/Cargo.toml | 5 + tools/iam_migrations/ldap/Cargo.toml | 5 + tools/orca/Cargo.toml | 2 +- unix_integration/resolver/Cargo.toml | 8 +- .../resolver/tests/cache_layer_test.rs | 3 + 43 files changed, 386 insertions(+), 370 deletions(-) rename server/testkit/{tests => defunct_tests}/oauth2_device_flow.rs (100%) create mode 100644 server/testkit/tests/DO_NOT_ADD_MORE_TESTS_HERE create mode 100644 server/testkit/tests/integration_test.rs delete mode 100644 server/testkit/tests/self.rs rename server/testkit/tests/{ => testkit}/apidocs.rs (76%) rename server/testkit/tests/{ => testkit}/domain.rs (100%) rename server/testkit/tests/{ => testkit}/group.rs (92%) rename server/testkit/tests/{ => testkit}/http_manifest.rs (67%) rename server/testkit/tests/{ => testkit}/https_extractors.rs (82%) rename server/testkit/tests/{ => testkit}/https_middleware.rs (67%) rename server/testkit/tests/{ => testkit}/identity_verification_tests.rs (97%) rename server/testkit/tests/{ => testkit}/integration.rs (100%) create mode 100644 server/testkit/tests/testkit/mod.rs rename server/testkit/tests/{ => testkit}/mtls_test.rs (100%) rename server/testkit/tests/{ => testkit}/oauth2_test.rs (96%) rename server/testkit/tests/{ => testkit}/person.rs (93%) rename server/testkit/tests/{ => testkit}/proto_v1_test.rs (99%) rename server/testkit/tests/{ => testkit}/scim_test.rs (74%) rename server/testkit/tests/{ => testkit}/service_account.rs (88%) rename server/testkit/tests/{ => testkit}/system.rs (86%) rename server/testkit/tests/{ => testkit}/unix.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index c61a05c3f..20c0fccd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3247,7 +3247,6 @@ dependencies = [ "escargot", "fantoccini", "futures", - "http 1.2.0", "jsonschema", "kanidm_build_profiles", "kanidm_client", diff --git a/Cargo.toml b/Cargo.toml index 588241cc1..298955995 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -241,6 +241,7 @@ reqwest = { version = "0.12.12", default-features = false, features = [ "json", "gzip", "rustls-tls-native-roots", + "rustls-tls-native-roots-no-provider", ] } rusqlite = { version = "^0.28.0", features = ["array", "bundled"] } rustls = { version = "0.23.21", default-features = false, features = [ diff --git a/libs/client/src/lib.rs b/libs/client/src/lib.rs index ee597b2b0..f389a1110 100644 --- a/libs/client/src/lib.rs +++ b/libs/client/src/lib.rs @@ -27,6 +27,7 @@ use std::time::Duration; use compact_jwt::Jwk; +pub use http; use kanidm_proto::constants::uri::V1_AUTH_VALID; use kanidm_proto::constants::{ ATTR_DOMAIN_DISPLAY_NAME, ATTR_DOMAIN_LDAP_BASEDN, ATTR_DOMAIN_SSID, ATTR_ENTRY_MANAGED_BY, @@ -137,6 +138,7 @@ pub struct KanidmClientBuilder { use_system_proxies: bool, /// Where to store auth tokens, only use in testing! token_cache_path: Option<String>, + disable_system_ca_store: bool, } impl Display for KanidmClientBuilder { @@ -170,33 +172,6 @@ impl Display for KanidmClientBuilder { } } -#[test] -fn test_kanidmclientbuilder_display() { - let defaultclient = KanidmClientBuilder::default(); - println!("{}", defaultclient); - assert!(defaultclient.to_string().contains("verify_ca")); - - let testclient = KanidmClientBuilder { - address: Some("https://example.com".to_string()), - verify_ca: true, - verify_hostnames: true, - ca: None, - connect_timeout: Some(420), - request_timeout: Some(69), - use_system_proxies: true, - token_cache_path: Some(CLIENT_TOKEN_CACHE.to_string()), - }; - println!("testclient {}", testclient); - assert!(testclient.to_string().contains("verify_ca: true")); - assert!(testclient.to_string().contains("verify_hostnames: true")); - - let badness = testclient.danger_accept_invalid_hostnames(true); - let badness = badness.danger_accept_invalid_certs(true); - println!("badness: {}", badness); - assert!(badness.to_string().contains("verify_ca: false")); - assert!(badness.to_string().contains("verify_hostnames: false")); -} - #[derive(Debug)] pub struct KanidmClient { pub(crate) client: reqwest::Client, @@ -233,6 +208,7 @@ impl KanidmClientBuilder { request_timeout: None, use_system_proxies: true, token_cache_path: None, + disable_system_ca_store: false, } } @@ -290,6 +266,7 @@ impl KanidmClientBuilder { request_timeout, use_system_proxies, token_cache_path, + disable_system_ca_store, } = self; // Process and apply all our options if they exist. let address = match kcc.uri { @@ -316,6 +293,7 @@ impl KanidmClientBuilder { request_timeout, use_system_proxies, token_cache_path, + disable_system_ca_store, }) } @@ -416,6 +394,16 @@ impl KanidmClientBuilder { } } + /// Enable or disable the native ca roots. By default these roots are enabled. + pub fn enable_native_ca_roots(self, enable: bool) -> Self { + KanidmClientBuilder { + // We have to flip the bool state here due to Default on bool being false + // and we want our options to be positive to a native speaker. + disable_system_ca_store: !enable, + ..self + } + } + pub fn danger_accept_invalid_hostnames(self, accept_invalid_hostnames: bool) -> Self { KanidmClientBuilder { // We have to flip the bool state here due to english language. @@ -527,6 +515,7 @@ impl KanidmClientBuilder { // implement sticky sessions with cookies. .cookie_store(true) .cookie_provider(client_cookies.clone()) + .tls_built_in_native_certs(!self.disable_system_ca_store) .danger_accept_invalid_hostnames(!self.verify_hostnames) .danger_accept_invalid_certs(!self.verify_ca); @@ -579,32 +568,6 @@ impl KanidmClientBuilder { } } -#[test] -fn test_make_url() { - use kanidm_proto::constants::DEFAULT_SERVER_ADDRESS; - let client: KanidmClient = KanidmClientBuilder::new() - .address(format!("https://{}", DEFAULT_SERVER_ADDRESS)) - .build() - .unwrap(); - assert_eq!( - client.get_url(), - Url::parse(&format!("https://{}", DEFAULT_SERVER_ADDRESS)).unwrap() - ); - assert_eq!( - client.make_url("/hello"), - Url::parse(&format!("https://{}/hello", DEFAULT_SERVER_ADDRESS)).unwrap() - ); - - let client: KanidmClient = KanidmClientBuilder::new() - .address(format!("https://{}/cheese/", DEFAULT_SERVER_ADDRESS)) - .build() - .unwrap(); - assert_eq!( - client.make_url("hello"), - Url::parse(&format!("https://{}/cheese/hello", DEFAULT_SERVER_ADDRESS)).unwrap() - ); -} - /// This is probably pretty jank but it works and was pulled from here: /// <https://github.com/seanmonstar/reqwest/issues/1602#issuecomment-1220996681> fn find_reqwest_error_source<E: std::error::Error + 'static>( @@ -623,6 +586,11 @@ fn find_reqwest_error_source<E: std::error::Error + 'static>( } impl KanidmClient { + /// Access the underlying reqwest client that has been configured for this Kanidm server + pub fn client(&self) -> &reqwest::Client { + &self.client + } + pub fn get_origin(&self) -> &Url { &self.origin } @@ -2174,31 +2142,97 @@ impl KanidmClient { } } -#[tokio::test] -async fn test_no_client_version_check_on_502() { - let res = reqwest::Response::from( - http::Response::builder() - .status(StatusCode::GATEWAY_TIMEOUT) - .body("") - .unwrap(), - ); - let client = KanidmClientBuilder::new() - .address("http://localhost:8080".to_string()) - .build() - .expect("Failed to build client"); - eprintln!("This should pass because we are returning 504 and shouldn't check version..."); - client.expect_version(&res).await; +#[cfg(test)] +mod tests { + use super::{KanidmClient, KanidmClientBuilder}; + use kanidm_proto::constants::CLIENT_TOKEN_CACHE; + use reqwest::StatusCode; + use url::Url; - let res = reqwest::Response::from( - http::Response::builder() - .status(StatusCode::BAD_GATEWAY) - .body("") - .unwrap(), - ); - let client = KanidmClientBuilder::new() - .address("http://localhost:8080".to_string()) - .build() - .expect("Failed to build client"); - eprintln!("This should pass because we are returning 502 and shouldn't check version..."); - client.expect_version(&res).await; + #[tokio::test] + async fn test_no_client_version_check_on_502() { + let res = reqwest::Response::from( + http::Response::builder() + .status(StatusCode::GATEWAY_TIMEOUT) + .body("") + .unwrap(), + ); + let client = KanidmClientBuilder::new() + .address("http://localhost:8080".to_string()) + .enable_native_ca_roots(false) + .build() + .expect("Failed to build client"); + eprintln!("This should pass because we are returning 504 and shouldn't check version..."); + client.expect_version(&res).await; + + let res = reqwest::Response::from( + http::Response::builder() + .status(StatusCode::BAD_GATEWAY) + .body("") + .unwrap(), + ); + let client = KanidmClientBuilder::new() + .address("http://localhost:8080".to_string()) + .enable_native_ca_roots(false) + .build() + .expect("Failed to build client"); + eprintln!("This should pass because we are returning 502 and shouldn't check version..."); + client.expect_version(&res).await; + } + + #[test] + fn test_make_url() { + use kanidm_proto::constants::DEFAULT_SERVER_ADDRESS; + let client: KanidmClient = KanidmClientBuilder::new() + .address(format!("https://{}", DEFAULT_SERVER_ADDRESS)) + .enable_native_ca_roots(false) + .build() + .unwrap(); + assert_eq!( + client.get_url(), + Url::parse(&format!("https://{}", DEFAULT_SERVER_ADDRESS)).unwrap() + ); + assert_eq!( + client.make_url("/hello"), + Url::parse(&format!("https://{}/hello", DEFAULT_SERVER_ADDRESS)).unwrap() + ); + + let client: KanidmClient = KanidmClientBuilder::new() + .address(format!("https://{}/cheese/", DEFAULT_SERVER_ADDRESS)) + .enable_native_ca_roots(false) + .build() + .unwrap(); + assert_eq!( + client.make_url("hello"), + Url::parse(&format!("https://{}/cheese/hello", DEFAULT_SERVER_ADDRESS)).unwrap() + ); + } + + #[test] + fn test_kanidmclientbuilder_display() { + let defaultclient = KanidmClientBuilder::default(); + println!("{}", defaultclient); + assert!(defaultclient.to_string().contains("verify_ca")); + + let testclient = KanidmClientBuilder { + address: Some("https://example.com".to_string()), + verify_ca: true, + verify_hostnames: true, + ca: None, + connect_timeout: Some(420), + request_timeout: Some(69), + use_system_proxies: true, + token_cache_path: Some(CLIENT_TOKEN_CACHE.to_string()), + disable_system_ca_store: false, + }; + println!("testclient {}", testclient); + assert!(testclient.to_string().contains("verify_ca: true")); + assert!(testclient.to_string().contains("verify_hostnames: true")); + + let badness = testclient.danger_accept_invalid_hostnames(true); + let badness = badness.danger_accept_invalid_certs(true); + println!("badness: {}", badness); + assert!(badness.to_string().contains("verify_ca: false")); + assert!(badness.to_string().contains("verify_hostnames: false")); + } } diff --git a/server/core/src/lib.rs b/server/core/src/lib.rs index 4826ec83f..b956735cd 100644 --- a/server/core/src/lib.rs +++ b/server/core/src/lib.rs @@ -115,9 +115,9 @@ async fn setup_qs_idms( .await?; // We generate a SINGLE idms only! - + let is_integration_test = config.integration_test_config.is_some(); let (idms, idms_delayed, idms_audit) = - IdmServer::new(query_server.clone(), &config.origin).await?; + IdmServer::new(query_server.clone(), &config.origin, is_integration_test).await?; Ok((query_server, idms, idms_delayed, idms_audit)) } diff --git a/server/daemon/Cargo.toml b/server/daemon/Cargo.toml index 24d710441..4f4b385cd 100644 --- a/server/daemon/Cargo.toml +++ b/server/daemon/Cargo.toml @@ -16,7 +16,7 @@ repository = { workspace = true } [[bin]] name = "kanidmd" path = "src/main.rs" -test = true +test = false doctest = false [features] diff --git a/server/lib-macros/Cargo.toml b/server/lib-macros/Cargo.toml index 9891a0393..85dafe444 100644 --- a/server/lib-macros/Cargo.toml +++ b/server/lib-macros/Cargo.toml @@ -13,7 +13,7 @@ repository = { workspace = true } [lib] proc-macro = true -test = true +test = false doctest = false [dependencies] diff --git a/server/lib/src/be/idl_arc_sqlite.rs b/server/lib/src/be/idl_arc_sqlite.rs index 6fb5d1a3f..ebc208827 100644 --- a/server/lib/src/be/idl_arc_sqlite.rs +++ b/server/lib/src/be/idl_arc_sqlite.rs @@ -18,7 +18,8 @@ use crate::be::idl_sqlite::{ IdlSqlite, IdlSqliteReadTransaction, IdlSqliteTransaction, IdlSqliteWriteTransaction, }; use crate::be::idxkey::{ - IdlCacheKey, IdlCacheKeyRef, IdlCacheKeyToRef, IdxKey, IdxKeyRef, IdxKeyToRef, IdxSlope, + IdlCacheKey, IdlCacheKeyRef, IdlCacheKeyToRef, IdxKey, IdxKeyRef, IdxKeyToRef, IdxNameKey, + IdxSlope, }; use crate::be::keystorage::{KeyHandle, KeyHandleId}; use crate::be::{BackendConfig, IdList, IdRawEntry}; @@ -35,6 +36,10 @@ const DEFAULT_NAME_CACHE_RATIO: usize = 8; const DEFAULT_CACHE_RMISS: usize = 0; const DEFAULT_CACHE_WMISS: usize = 0; +const DEFAULT_IDX_CACHE_RMISS: usize = 8; +const DEFAULT_IDX_CACHE_WMISS: usize = 16; +const DEFAULT_IDX_EXISTS_TARGET: usize = 256; + #[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] enum NameCacheKey { Name2Uuid(String), @@ -55,6 +60,9 @@ pub struct IdlArcSqlite { entry_cache: ARCache<u64, Arc<EntrySealedCommitted>>, idl_cache: ARCache<IdlCacheKey, Box<IDLBitRange>>, name_cache: ARCache<NameCacheKey, NameCacheValue>, + + idx_exists_cache: ARCache<IdxNameKey, bool>, + op_ts_max: CowCell<Option<Duration>>, allids: CowCell<IDLBitRange>, maxid: CowCell<u64>, @@ -66,6 +74,8 @@ pub struct IdlArcSqliteReadTransaction<'a> { entry_cache: ARCacheReadTxn<'a, u64, Arc<EntrySealedCommitted>, ()>, idl_cache: ARCacheReadTxn<'a, IdlCacheKey, Box<IDLBitRange>, ()>, name_cache: ARCacheReadTxn<'a, NameCacheKey, NameCacheValue, ()>, + + idx_exists_cache: ARCacheReadTxn<'a, IdxNameKey, bool, ()>, allids: CowCellReadTxn<IDLBitRange>, } @@ -74,6 +84,9 @@ pub struct IdlArcSqliteWriteTransaction<'a> { entry_cache: ARCacheWriteTxn<'a, u64, Arc<EntrySealedCommitted>, ()>, idl_cache: ARCacheWriteTxn<'a, IdlCacheKey, Box<IDLBitRange>, ()>, name_cache: ARCacheWriteTxn<'a, NameCacheKey, NameCacheValue, ()>, + + idx_exists_cache: ARCacheWriteTxn<'a, IdxNameKey, bool, ()>, + op_ts_max: CowCellWriteTxn<'a, Option<Duration>>, allids: CowCellWriteTxn<'a, IDLBitRange>, maxid: CowCellWriteTxn<'a, u64>, @@ -178,8 +191,8 @@ macro_rules! get_idl { // or smaller type. Perhaps even a small cache of the IdlCacheKeys that // are allocated to reduce some allocs? Probably over thinking it at // this point. - // - // First attempt to get from this cache. + + // Now attempt to get from this cache. let cache_key = IdlCacheKeyRef { a: $attr, i: $itype, @@ -195,16 +208,47 @@ macro_rules! get_idl { ); return Ok(Some(data.as_ref().clone())); } + + // If it was a miss, does the actually exist in the DB? + let idx_key = IdxNameKey { + a: $attr.clone(), + i: $itype, + }; + let idx_r = $self.idx_exists_cache.get(&idx_key); + if idx_r == Some(&false) { + // The idx does not exist - bail early. + return Ok(None) + } + + // The table either exists and we don't have data on it yet, + // or it does not exist and we need to hear back from the lower level + // If miss, get from db *and* insert to the cache. let db_r = $self.db.get_idl($attr, $itype, $idx_key)?; + if let Some(ref idl) = db_r { + if idx_r == None { + // It exists, so track that data, because we weren't + // previously tracking it. + $self.idx_exists_cache.insert(idx_key, true) + } + let ncache_key = IdlCacheKey { a: $attr.clone(), i: $itype.clone(), k: $idx_key.into(), }; $self.idl_cache.insert(ncache_key, Box::new(idl.clone())) - } + } else { + // The DB was unable to return this idx because table backing the + // idx does not exist. We should cache this to prevent repeat hits + // on sqlite until the db does exist, at which point the cache is + // cleared anyway. + // + // NOTE: If the db idx misses it returns Some(empty_set), so this + // only caches missing index tables. + $self.idx_exists_cache.insert(idx_key, false) + }; Ok(db_r) }}; } @@ -593,6 +637,7 @@ impl IdlArcSqliteWriteTransaction<'_> { */ self.entry_cache.clear(); self.idl_cache.clear(); + self.idx_exists_cache.clear(); self.name_cache.clear(); Ok(()) } @@ -604,6 +649,7 @@ impl IdlArcSqliteWriteTransaction<'_> { mut entry_cache, mut idl_cache, mut name_cache, + idx_exists_cache, op_ts_max, allids, maxid, @@ -677,6 +723,7 @@ impl IdlArcSqliteWriteTransaction<'_> { // Can no longer fail from this point. op_ts_max.commit(); name_cache.commit(); + idx_exists_cache.commit(); idl_cache.commit(); allids.commit(); maxid.commit(); @@ -708,6 +755,7 @@ impl IdlArcSqliteWriteTransaction<'_> { *self.maxid = mid; } + #[instrument(level = "trace", skip_all)] pub fn write_identries<'b, I>(&'b mut self, mut entries: I) -> Result<(), OperationError> where I: Iterator<Item = &'b Entry<EntrySealed, EntryCommitted>>, @@ -757,6 +805,7 @@ impl IdlArcSqliteWriteTransaction<'_> { }) } + #[instrument(level = "trace", skip_all)] pub fn write_idl( &mut self, attr: &Attribute, @@ -1127,9 +1176,17 @@ impl IdlArcSqliteWriteTransaction<'_> { Ok(()) } - pub fn create_idx(&self, attr: &Attribute, itype: IndexType) -> Result<(), OperationError> { - // We don't need to affect this, so pass it down. - self.db.create_idx(attr, itype) + pub fn create_idx(&mut self, attr: &Attribute, itype: IndexType) -> Result<(), OperationError> { + self.db.create_idx(attr, itype)?; + + // Cache that this exists since we just made it. + let idx_key = IdxNameKey { + a: attr.clone(), + i: itype, + }; + self.idx_exists_cache.insert(idx_key, true); + + Ok(()) } /// ⚠️ - This function will destroy all indexes in the database. @@ -1141,6 +1198,7 @@ impl IdlArcSqliteWriteTransaction<'_> { debug!("CLEARING CACHE"); self.db.danger_purge_idxs().map(|()| { self.idl_cache.clear(); + self.idx_exists_cache.clear(); self.name_cache.clear(); }) } @@ -1266,6 +1324,21 @@ impl IdlArcSqlite { OperationError::InvalidState })?; + let idx_exists_cache = ARCacheBuilder::new() + .set_expected_workload( + DEFAULT_IDX_EXISTS_TARGET, + cfg.pool_size as usize, + DEFAULT_IDX_CACHE_RMISS, + DEFAULT_IDX_CACHE_WMISS, + true, + ) + .set_reader_quiesce(true) + .build() + .ok_or_else(|| { + admin_error!("Failed to construct idx_exists_cache"); + OperationError::InvalidState + })?; + let allids = CowCell::new(IDLBitRange::new()); let maxid = CowCell::new(0); @@ -1279,6 +1352,7 @@ impl IdlArcSqlite { entry_cache, idl_cache, name_cache, + idx_exists_cache, op_ts_max, allids, maxid, @@ -1298,6 +1372,7 @@ impl IdlArcSqlite { let db_read = self.db.read()?; let idl_cache_read = self.idl_cache.read(); let name_cache_read = self.name_cache.read(); + let idx_exists_cache_read = self.idx_exists_cache.read(); let allids_read = self.allids.read(); Ok(IdlArcSqliteReadTransaction { @@ -1305,6 +1380,7 @@ impl IdlArcSqlite { entry_cache: entry_cache_read, idl_cache: idl_cache_read, name_cache: name_cache_read, + idx_exists_cache: idx_exists_cache_read, allids: allids_read, }) } @@ -1315,6 +1391,7 @@ impl IdlArcSqlite { let db_write = self.db.write()?; let idl_cache_write = self.idl_cache.write(); let name_cache_write = self.name_cache.write(); + let idx_exists_cache_write = self.idx_exists_cache.write(); let op_ts_max_write = self.op_ts_max.write(); let allids_write = self.allids.write(); let maxid_write = self.maxid.write(); @@ -1325,6 +1402,7 @@ impl IdlArcSqlite { entry_cache: entry_cache_write, idl_cache: idl_cache_write, name_cache: name_cache_write, + idx_exists_cache: idx_exists_cache_write, op_ts_max: op_ts_max_write, allids: allids_write, maxid: maxid_write, diff --git a/server/lib/src/be/idl_sqlite.rs b/server/lib/src/be/idl_sqlite.rs index 44ab11ad5..a1c5ab377 100644 --- a/server/lib/src/be/idl_sqlite.rs +++ b/server/lib/src/be/idl_sqlite.rs @@ -205,12 +205,15 @@ pub(crate) trait IdlSqliteTransaction { let mut stmt = self .get_conn()? .prepare(&format!( - "SELECT COUNT(name) from {}.sqlite_master where name = :tname", + "SELECT rowid from {}.sqlite_master where name = :tname LIMIT 1", self.get_db_name() )) .map_err(sqlite_error)?; + let i: Option<i64> = stmt .query_row(&[(":tname", tname)], |row| row.get(0)) + // If the row doesn't exist, we don't mind. + .optional() .map_err(sqlite_error)?; match i { diff --git a/server/lib/src/be/idxkey.rs b/server/lib/src/be/idxkey.rs index a0f5c5caf..945717bf0 100644 --- a/server/lib/src/be/idxkey.rs +++ b/server/lib/src/be/idxkey.rs @@ -155,3 +155,9 @@ impl Ord for (dyn IdlCacheKeyToRef + '_) { self.keyref().cmp(&other.keyref()) } } + +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] +pub struct IdxNameKey { + pub a: Attribute, + pub i: IndexType, +} diff --git a/server/lib/src/idm/server.rs b/server/lib/src/idm/server.rs index 038c44fc5..ed55cf69c 100644 --- a/server/lib/src/idm/server.rs +++ b/server/lib/src/idm/server.rs @@ -143,8 +143,9 @@ impl IdmServer { pub async fn new( qs: QueryServer, origin: &str, + is_integration_test: bool, ) -> Result<(IdmServer, IdmServerDelayed, IdmServerAudit), OperationError> { - let crypto_policy = if cfg!(test) { + let crypto_policy = if cfg!(test) || is_integration_test { CryptoPolicy::danger_test_minimum() } else { // This is calculated back from: diff --git a/server/lib/src/plugins/memberof.rs b/server/lib/src/plugins/memberof.rs index ff266a808..a8b980c9e 100644 --- a/server/lib/src/plugins/memberof.rs +++ b/server/lib/src/plugins/memberof.rs @@ -571,35 +571,43 @@ impl Plugin for MemberOf { Err(e) => return vec![e], }; + // First we have to build a direct membership map. This saves us + // needing to run queries since we already have every entry on hand + // from the all_cand search. + let mut direct_membership_map: BTreeMap<Uuid, BTreeSet<Uuid>> = Default::default(); + + let pv_class: PartialValue = EntryClass::Group.into(); + + for entry in all_cand.iter() { + if !entry.attribute_equality(Attribute::Class, &pv_class) { + // Not a group, move on. + continue; + } + + let group_uuid = entry.get_uuid(); + + let member_iter = entry + .get_ava_refer(Attribute::Member) + .into_iter() + .flat_map(|set| set.iter()) + .chain( + entry + .get_ava_refer(Attribute::DynMember) + .into_iter() + .flat_map(|set| set.iter()), + ); + + for member_uuid in member_iter { + let member_groups = direct_membership_map.entry(*member_uuid).or_default(); + member_groups.insert(group_uuid); + } + } + // for each entry in the DB (live). for e in all_cand { let uuid = e.get_uuid(); - let filt_in = filter!(f_and!([ - f_eq(Attribute::Class, EntryClass::Group.into()), - f_or!([ - f_eq(Attribute::Member, PartialValue::Refer(uuid)), - f_eq(Attribute::DynMember, PartialValue::Refer(uuid)) - ]) - ])); - // what groups is this entry a direct member of? - let direct_memberof = match qs - .internal_search(filt_in) - .map_err(|_| ConsistencyError::QueryServerSearchFailure) - { - Ok(d_mo) => d_mo, - Err(e) => return vec![Err(e)], - }; - - // for all direct -> add uuid to map - let d_groups_set: BTreeSet<Uuid> = - direct_memberof.iter().map(|e| e.get_uuid()).collect(); - - let d_groups_set = if d_groups_set.is_empty() { - None - } else { - Some(d_groups_set) - }; + let d_groups_set: Option<&BTreeSet<Uuid>> = direct_membership_map.get(&uuid); trace!( "DMO search groups {:?} -> {:?}", @@ -607,12 +615,16 @@ impl Plugin for MemberOf { d_groups_set ); + // Remember, we only need to check direct memberships, because when memberof + // it applies it clones dmo -> mo, so validation of all dmo sets implies mo is + // valid (and a subset) of dmo. + match (e.get_ava_set(Attribute::DirectMemberOf), d_groups_set) { (Some(edmos), Some(b)) => { // Can they both be reference sets? match edmos.as_refer_set() { Some(a) => { - let diff: Vec<_> = a.symmetric_difference(&b).collect(); + let diff: Vec<_> = a.symmetric_difference(b).collect(); if !diff.is_empty() { error!( "MemberOfInvalid: Entry {}, DMO has inconsistencies", @@ -636,7 +648,7 @@ impl Plugin for MemberOf { } (entry_direct_member_of, expected_direct_groups) => { error!( - "MemberOfInvalid directmemberof set and DMO search set differ in size: {}", + "MemberOfInvalid directmemberof set and DMO search set differ in presence: {}", e.get_display_id() ); // trace!(?e); @@ -645,19 +657,6 @@ impl Plugin for MemberOf { r.push(Err(ConsistencyError::MemberOfInvalid(e.get_id()))); } } - - // Could check all dmos in mos? - - /* To check nested! */ - // add all direct to a stack - // for all in stack - // check their direct memberships - // if not in map - // add to map - // push to stack - - // check mo == map set - // if not, consistency error! } r diff --git a/server/lib/src/testkit.rs b/server/lib/src/testkit.rs index 239009e70..98342020b 100644 --- a/server/lib/src/testkit.rs +++ b/server/lib/src/testkit.rs @@ -89,7 +89,7 @@ pub async fn setup_idm_test( ) -> (IdmServer, IdmServerDelayed, IdmServerAudit) { let qs = setup_test(config).await; - IdmServer::new(qs, "https://idm.example.com") + IdmServer::new(qs, "https://idm.example.com", true) .await .expect("Failed to setup idms") } diff --git a/server/testkit-macros/Cargo.toml b/server/testkit-macros/Cargo.toml index 594b41382..cb1af8006 100644 --- a/server/testkit-macros/Cargo.toml +++ b/server/testkit-macros/Cargo.toml @@ -5,7 +5,7 @@ edition = { workspace = true } [lib] proc-macro = true -test = true +test = false doctest = false [dependencies] diff --git a/server/testkit/Cargo.toml b/server/testkit/Cargo.toml index acc107b23..21f0498f1 100644 --- a/server/testkit/Cargo.toml +++ b/server/testkit/Cargo.toml @@ -14,7 +14,7 @@ repository = { workspace = true } [lib] name = "kanidmd_testkit" path = "src/lib.rs" -test = true +test = false doctest = false [features] @@ -25,7 +25,6 @@ webdriver = [] dev-oauth2-device-flow = [] [dependencies] -http = { workspace = true } kanidm_client = { workspace = true } kanidm_proto = { workspace = true } kanidmd_core = { workspace = true } @@ -57,7 +56,6 @@ futures = { workspace = true } oauth2_ext = { workspace = true, default-features = false, features = [ "reqwest", ] } -openssl = { workspace = true } petgraph = { version = "0.7.1", features = ["serde"] } serde_json = { workspace = true } time = { workspace = true } diff --git a/server/testkit/tests/oauth2_device_flow.rs b/server/testkit/defunct_tests/oauth2_device_flow.rs similarity index 100% rename from server/testkit/tests/oauth2_device_flow.rs rename to server/testkit/defunct_tests/oauth2_device_flow.rs diff --git a/server/testkit/src/lib.rs b/server/testkit/src/lib.rs index 1d4e644ca..6dd4677dd 100644 --- a/server/testkit/src/lib.rs +++ b/server/testkit/src/lib.rs @@ -92,6 +92,7 @@ pub async fn setup_async_test(mut config: Configuration) -> (KanidmClient, CoreH #[allow(clippy::panic)] let rsclient = match KanidmClientBuilder::new() .address(addr.clone()) + .enable_native_ca_roots(false) .no_proxy() .build() { @@ -396,7 +397,7 @@ macro_rules! assert_no_cache { // Check we have correct nocache headers. let cache_header: &str = $response .headers() - .get(http::header::CACHE_CONTROL) + .get(kanidm_client::http::header::CACHE_CONTROL) .expect("missing cache-control header") .to_str() .expect("invalid cache-control header"); diff --git a/server/testkit/tests/DO_NOT_ADD_MORE_TESTS_HERE b/server/testkit/tests/DO_NOT_ADD_MORE_TESTS_HERE new file mode 100644 index 000000000..247376ef4 --- /dev/null +++ b/server/testkit/tests/DO_NOT_ADD_MORE_TESTS_HERE @@ -0,0 +1,11 @@ + +DO NOT ADD MORE TESTS TO THIS DIRECTORY + +Add them to the `testkit` directory. + +Each new test in this folder is a separate binary, that must be complied linked and executed. This +takes HUGES AMOUNTS OF TIME. It makes tests unbelievably slow. + +If you want to add an integration test, put it into the testkit dir so it becomes part of the +single larger integration test runner. + diff --git a/server/testkit/tests/integration_test.rs b/server/testkit/tests/integration_test.rs new file mode 100644 index 000000000..1a6f6f52e --- /dev/null +++ b/server/testkit/tests/integration_test.rs @@ -0,0 +1,3 @@ +#![deny(warnings)] + +mod testkit; diff --git a/server/testkit/tests/self.rs b/server/testkit/tests/self.rs deleted file mode 100644 index 86d373657..000000000 --- a/server/testkit/tests/self.rs +++ /dev/null @@ -1,57 +0,0 @@ -use kanidm_client::KanidmClient; - -/// This literally tests that the thing exists and responds in a way we expect, probably worth testing it better... -#[kanidmd_testkit::test] -async fn test_v1_self_applinks(rsclient: KanidmClient) { - // We need to do manual reqwests here. - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); - - let response = match client - .get(rsclient.make_url("/v1/self/_applinks")) - .send() - .await - { - Ok(value) => value, - Err(error) => { - panic!( - "Failed to query {:?} : {:#?}", - rsclient.make_url("/v1/self/_applinks"), - error - ); - } - }; - eprintln!("response: {:#?}", response); - assert_eq!(response.status(), 401); - - let body = response.text().await.unwrap(); - eprintln!("{}", body); -} - -/// This literally tests that the thing exists and responds in a way we expect, probably worth testing it better... -#[kanidmd_testkit::test] -async fn test_v1_self_whoami_uat(rsclient: KanidmClient) { - // We need to do manual reqwests here. - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); - - let response = match client.get(rsclient.make_url("/v1/self/_uat")).send().await { - Ok(value) => value, - Err(error) => { - panic!( - "Failed to query {:?} : {:#?}", - rsclient.make_url("/v1/self/_uat"), - error - ); - } - }; - eprintln!("response: {:#?}", response); - assert_eq!(response.status(), 401); - - let body = response.text().await.unwrap(); - eprintln!("{}", body); -} diff --git a/server/testkit/tests/apidocs.rs b/server/testkit/tests/testkit/apidocs.rs similarity index 76% rename from server/testkit/tests/apidocs.rs rename to server/testkit/tests/testkit/apidocs.rs index ca9a87602..1f1ef5d84 100644 --- a/server/testkit/tests/apidocs.rs +++ b/server/testkit/tests/testkit/apidocs.rs @@ -12,21 +12,18 @@ async fn check_that_the_swagger_api_loads(rsclient: kanidm_client::KanidmClient) rsclient.set_token("".into()).await; info!("Running test: check_that_the_swagger_api_loads"); let url = rsclient.make_url("/docs/v1/openapi.json"); - let openapi_response: OpenAPIResponse = reqwest::get(url.clone()) + + let openapi_response = rsclient + .perform_get_request::<OpenAPIResponse>(url.as_str()) .await - .expect("Failed to get openapi.json") - .json() - .await - .unwrap(); + .expect("Failed to get openapi.json"); assert_eq!(openapi_response.openapi, "3.0.3"); // this validates that it's valid JSON schema, but not that it's valid openapi... but it's a start. - let schema: serde_json::Value = reqwest::get(url) + let schema = rsclient + .perform_get_request::<serde_json::Value>(url.as_str()) .await - .expect("Failed to get openapi.json") - .json() - .await - .unwrap(); + .expect("Failed to get openapi.json"); let instance = serde_json::json!("foo"); let compiled = Validator::new(&schema).expect("A valid schema"); diff --git a/server/testkit/tests/domain.rs b/server/testkit/tests/testkit/domain.rs similarity index 100% rename from server/testkit/tests/domain.rs rename to server/testkit/tests/testkit/domain.rs diff --git a/server/testkit/tests/group.rs b/server/testkit/tests/testkit/group.rs similarity index 92% rename from server/testkit/tests/group.rs rename to server/testkit/tests/testkit/group.rs index 971fa081b..67bf05e28 100644 --- a/server/testkit/tests/group.rs +++ b/server/testkit/tests/testkit/group.rs @@ -1,4 +1,4 @@ -use kanidm_client::{ClientError, KanidmClient}; +use kanidm_client::{ClientError, KanidmClient, StatusCode}; use kanidm_proto::constants::ATTR_DESCRIPTION; use kanidmd_testkit::{create_user, ADMIN_TEST_PASSWORD, ADMIN_TEST_USER}; use serde_json::Value; @@ -48,6 +48,6 @@ async fn test_v1_group_id_attr_post(rsclient: KanidmClient) { eprintln!("response: {:#?}", response); assert!(matches!( response, - ClientError::Http(reqwest::StatusCode::BAD_REQUEST, _, _) + ClientError::Http(StatusCode::BAD_REQUEST, _, _) )); } diff --git a/server/testkit/tests/http_manifest.rs b/server/testkit/tests/testkit/http_manifest.rs similarity index 67% rename from server/testkit/tests/http_manifest.rs rename to server/testkit/tests/testkit/http_manifest.rs index 7e78c3e22..fb46d8f35 100644 --- a/server/testkit/tests/http_manifest.rs +++ b/server/testkit/tests/testkit/http_manifest.rs @@ -1,11 +1,16 @@ -use kanidm_client::KanidmClient; +use kanidm_client::{http::header, KanidmClient}; #[kanidmd_testkit::test] async fn test_https_manifest(rsclient: KanidmClient) { // We need to do manual reqwests here. + let client = rsclient.client(); // here we test the /ui/ endpoint which should have the headers - let response = match reqwest::get(rsclient.make_url("/manifest.webmanifest")).await { + let response = match client + .get(rsclient.make_url("/manifest.webmanifest")) + .send() + .await + { Ok(value) => value, Err(error) => { panic!( @@ -20,8 +25,6 @@ async fn test_https_manifest(rsclient: KanidmClient) { eprintln!( "csp headers: {:#?}", - response - .headers() - .get(http::header::CONTENT_SECURITY_POLICY) + response.headers().get(header::CONTENT_SECURITY_POLICY) ); } diff --git a/server/testkit/tests/https_extractors.rs b/server/testkit/tests/testkit/https_extractors.rs similarity index 82% rename from server/testkit/tests/https_extractors.rs rename to server/testkit/tests/testkit/https_extractors.rs index 15da716af..6fd6bb288 100644 --- a/server/testkit/tests/https_extractors.rs +++ b/server/testkit/tests/testkit/https_extractors.rs @@ -12,10 +12,8 @@ const DEFAULT_IP_ADDRESS: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); #[kanidmd_testkit::test(trust_x_forward_for = false)] async fn dont_trust_xff_send_header(rsclient: KanidmClient) { - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header( @@ -35,10 +33,8 @@ async fn dont_trust_xff_send_header(rsclient: KanidmClient) { #[kanidmd_testkit::test(trust_x_forward_for = false)] async fn dont_trust_xff_dont_send_header(rsclient: KanidmClient) { - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header( @@ -63,10 +59,8 @@ async fn dont_trust_xff_dont_send_header(rsclient: KanidmClient) { #[kanidmd_testkit::test(trust_x_forward_for = true)] async fn trust_xff_send_invalid_header_single_value(rsclient: KanidmClient) { - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header( @@ -85,10 +79,8 @@ async fn trust_xff_send_invalid_header_single_value(rsclient: KanidmClient) { // #[kanidmd_testkit::test(trust_x_forward_for = true)] async fn trust_xff_send_invalid_header_multiple_values(rsclient: KanidmClient) { - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header( @@ -106,10 +98,8 @@ async fn trust_xff_send_invalid_header_multiple_values(rsclient: KanidmClient) { async fn trust_xff_send_valid_header_single_ipv4_address(rsclient: KanidmClient) { let ip_addr = "2001:db8:85a3:8d3:1319:8a2e:370:7348"; - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header(X_FORWARDED_FOR, ip_addr) @@ -128,10 +118,8 @@ async fn trust_xff_send_valid_header_single_ipv4_address(rsclient: KanidmClient) async fn trust_xff_send_valid_header_single_ipv6_address(rsclient: KanidmClient) { let ip_addr = "203.0.113.195"; - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header(X_FORWARDED_FOR, ip_addr) @@ -150,10 +138,8 @@ async fn trust_xff_send_valid_header_single_ipv6_address(rsclient: KanidmClient) async fn trust_xff_send_valid_header_multiple_address(rsclient: KanidmClient) { let first_ip_addr = "203.0.113.195, 2001:db8:85a3:8d3:1319:8a2e:370:7348"; - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header(X_FORWARDED_FOR, first_ip_addr) @@ -172,10 +158,6 @@ async fn trust_xff_send_valid_header_multiple_address(rsclient: KanidmClient) { let second_ip_addr = "2001:db8:85a3:8d3:1319:8a2e:370:7348, 198.51.100.178, 203.0.113.195"; - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .header(X_FORWARDED_FOR, second_ip_addr) @@ -195,10 +177,8 @@ async fn trust_xff_send_valid_header_multiple_address(rsclient: KanidmClient) { #[kanidmd_testkit::test(trust_x_forward_for = true)] async fn trust_xff_dont_send_header(rsclient: KanidmClient) { - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); + let res = client .get(rsclient.make_url("/v1/debug/ipinfo")) .send() diff --git a/server/testkit/tests/https_middleware.rs b/server/testkit/tests/testkit/https_middleware.rs similarity index 67% rename from server/testkit/tests/https_middleware.rs rename to server/testkit/tests/testkit/https_middleware.rs index 4e02a9118..0d33d12f3 100644 --- a/server/testkit/tests/https_middleware.rs +++ b/server/testkit/tests/testkit/https_middleware.rs @@ -1,11 +1,13 @@ +use kanidm_client::http::header; use kanidm_client::KanidmClient; #[kanidmd_testkit::test] async fn test_https_middleware_headers(rsclient: KanidmClient) { // We need to do manual reqwests here. + let client = rsclient.client(); // here we test the /ui/ endpoint which should have the headers - let response = match reqwest::get(rsclient.make_url("/ui")).await { + let response = match client.get(rsclient.make_url("/ui")).send().await { Ok(value) => value, Err(error) => { panic!( @@ -19,19 +21,15 @@ async fn test_https_middleware_headers(rsclient: KanidmClient) { assert_eq!(response.status(), 200); eprintln!( "csp headers: {:#?}", - response - .headers() - .get(http::header::CONTENT_SECURITY_POLICY) + response.headers().get(header::CONTENT_SECURITY_POLICY) ); assert_ne!( - response - .headers() - .get(http::header::CONTENT_SECURITY_POLICY), + response.headers().get(header::CONTENT_SECURITY_POLICY), None ); // here we test the /ui/login endpoint which should have the headers - let response = match reqwest::get(rsclient.make_url("/ui/login")).await { + let response = match client.get(rsclient.make_url("/ui/login")).send().await { Ok(value) => value, Err(error) => { panic!( @@ -46,14 +44,10 @@ async fn test_https_middleware_headers(rsclient: KanidmClient) { eprintln!( "csp headers: {:#?}", - response - .headers() - .get(http::header::CONTENT_SECURITY_POLICY) + response.headers().get(header::CONTENT_SECURITY_POLICY) ); assert_ne!( - response - .headers() - .get(http::header::CONTENT_SECURITY_POLICY), + response.headers().get(header::CONTENT_SECURITY_POLICY), None ); } diff --git a/server/testkit/tests/identity_verification_tests.rs b/server/testkit/tests/testkit/identity_verification_tests.rs similarity index 97% rename from server/testkit/tests/identity_verification_tests.rs rename to server/testkit/tests/testkit/identity_verification_tests.rs index a4a05f35f..285c908ab 100644 --- a/server/testkit/tests/identity_verification_tests.rs +++ b/server/testkit/tests/testkit/identity_verification_tests.rs @@ -1,11 +1,9 @@ use core::result::Result::Err; -use kanidm_client::KanidmClient; +use kanidm_client::{KanidmClient, StatusCode}; use kanidm_proto::internal::OperationError; use kanidm_proto::internal::{IdentifyUserRequest, IdentifyUserResponse}; - use kanidmd_lib::prelude::Attribute; use kanidmd_testkit::ADMIN_TEST_PASSWORD; -use reqwest::StatusCode; static UNIVERSAL_PW: &str = "eicieY7ahchaoCh0eeTa"; @@ -26,14 +24,14 @@ async fn test_not_authenticated(rsclient: KanidmClient) { .idm_person_identify_user(USER_A_NAME, IdentifyUserRequest::Start) .await; assert!( - matches!(res, Err(err) if matches!(err, kanidm_client::ClientError::Http(reqwest::StatusCode::UNAUTHORIZED, ..))) + matches!(res, Err(err) if matches!(err, kanidm_client::ClientError::Http(StatusCode::UNAUTHORIZED, ..))) ); let res = rsclient .idm_person_identify_user(USER_A_NAME, IdentifyUserRequest::DisplayCode) .await; assert!( - matches!(res, Err(err) if matches!(err, kanidm_client::ClientError::Http(reqwest::StatusCode::UNAUTHORIZED, ..))) + matches!(res, Err(err) if matches!(err, kanidm_client::ClientError::Http(StatusCode::UNAUTHORIZED, ..))) ); let res = rsclient .idm_person_identify_user( @@ -43,7 +41,7 @@ async fn test_not_authenticated(rsclient: KanidmClient) { .await; assert!( - matches!(res, Err(err) if matches!(err, kanidm_client::ClientError::Http(reqwest::StatusCode::UNAUTHORIZED, ..))) + matches!(res, Err(err) if matches!(err, kanidm_client::ClientError::Http(StatusCode::UNAUTHORIZED, ..))) ); } diff --git a/server/testkit/tests/integration.rs b/server/testkit/tests/testkit/integration.rs similarity index 100% rename from server/testkit/tests/integration.rs rename to server/testkit/tests/testkit/integration.rs diff --git a/server/testkit/tests/testkit/mod.rs b/server/testkit/tests/testkit/mod.rs new file mode 100644 index 000000000..8c70d5166 --- /dev/null +++ b/server/testkit/tests/testkit/mod.rs @@ -0,0 +1,16 @@ +mod apidocs; +mod domain; +mod group; +mod http_manifest; +mod https_extractors; +mod https_middleware; +mod identity_verification_tests; +mod integration; +mod mtls_test; +mod oauth2_test; +mod person; +mod proto_v1_test; +mod scim_test; +mod service_account; +mod system; +mod unix; diff --git a/server/testkit/tests/mtls_test.rs b/server/testkit/tests/testkit/mtls_test.rs similarity index 100% rename from server/testkit/tests/mtls_test.rs rename to server/testkit/tests/testkit/mtls_test.rs diff --git a/server/testkit/tests/oauth2_test.rs b/server/testkit/tests/testkit/oauth2_test.rs similarity index 96% rename from server/testkit/tests/oauth2_test.rs rename to server/testkit/tests/testkit/oauth2_test.rs index 500745567..80cd2d083 100644 --- a/server/testkit/tests/oauth2_test.rs +++ b/server/testkit/tests/testkit/oauth2_test.rs @@ -16,11 +16,10 @@ use kanidmd_lib::constants::NAME_IDM_ALL_ACCOUNTS; use kanidmd_lib::prelude::Attribute; use oauth2_ext::PkceCodeChallenge; use reqwest::header::{HeaderValue, CONTENT_TYPE}; -use reqwest::StatusCode; use uri::{OAUTH2_TOKEN_ENDPOINT, OAUTH2_TOKEN_INTROSPECT_ENDPOINT, OAUTH2_TOKEN_REVOKE_ENDPOINT}; use url::{form_urlencoded::parse as query_parse, Url}; -use kanidm_client::KanidmClient; +use kanidm_client::{http::header, KanidmClient, StatusCode}; use kanidmd_testkit::{ assert_no_cache, ADMIN_TEST_PASSWORD, ADMIN_TEST_USER, NOT_ADMIN_TEST_EMAIL, NOT_ADMIN_TEST_PASSWORD, NOT_ADMIN_TEST_USERNAME, TEST_INTEGRATION_RS_DISPLAY, @@ -137,6 +136,7 @@ async fn test_oauth2_openid_basic_flow_impl( // from here, we can now begin what would be a "interaction" to the oauth server. // Create a new reqwest client - we'll be using this manually. let client = reqwest::Client::builder() + .tls_built_in_native_certs(false) .redirect(reqwest::redirect::Policy::none()) .no_proxy() .build() @@ -152,11 +152,11 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send discovery preflight request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); let cors_header: &str = response .headers() - .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN) + .get(header::ACCESS_CONTROL_ALLOW_ORIGIN) .expect("missing access-control-allow-origin header") .to_str() .expect("invalid access-control-allow-origin header"); @@ -168,12 +168,12 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); // Assert CORS on the GET too. let cors_header: &str = response .headers() - .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN) + .get(header::ACCESS_CONTROL_ALLOW_ORIGIN) .expect("missing access-control-allow-origin header") .to_str() .expect("invalid access-control-allow-origin header"); @@ -221,7 +221,7 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); assert_no_cache!(response); let mut jwk_set: JwkKeySet = response @@ -264,7 +264,7 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); assert_no_cache!(response); let consent_req: AuthorisationResponse = response @@ -298,7 +298,7 @@ async fn test_oauth2_openid_basic_flow_impl( .expect("Failed to send request."); // This should yield a 302 redirect with some query params. - assert_eq!(response.status(), reqwest::StatusCode::FOUND); + assert_eq!(response.status(), StatusCode::FOUND); assert_no_cache!(response); // And we should have a URL in the location header. @@ -343,11 +343,11 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send code exchange request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); let cors_header: &str = response .headers() - .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN) + .get(header::ACCESS_CONTROL_ALLOW_ORIGIN) .expect("missing access-control-allow-origin header") .to_str() .expect("invalid access-control-allow-origin header"); @@ -379,7 +379,7 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send token introspect request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); tracing::trace!("{:?}", response.headers()); assert!( response.headers().get(CONTENT_TYPE) == Some(&HeaderValue::from_static(APPLICATION_JSON)) @@ -485,7 +485,7 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send client credentials request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); let atr = response .json::<AccessTokenResponse>() @@ -506,7 +506,7 @@ async fn test_oauth2_openid_basic_flow_impl( .await .expect("Failed to send token introspect request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); let tir = response .json::<AccessTokenIntrospectResponse>() @@ -681,6 +681,7 @@ async fn test_oauth2_openid_public_flow_impl( // Create a new reqwest client - we'll be using this manually. let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) + .tls_built_in_native_certs(false) .no_proxy() .build() .expect("Failed to create client."); @@ -692,7 +693,7 @@ async fn test_oauth2_openid_public_flow_impl( .await .expect("Failed to send request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); assert_no_cache!(response); let mut jwk_set: JwkKeySet = response @@ -733,7 +734,7 @@ async fn test_oauth2_openid_public_flow_impl( .await .expect("Failed to send request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); assert_no_cache!(response); let consent_req: AuthorisationResponse = response @@ -765,7 +766,7 @@ async fn test_oauth2_openid_public_flow_impl( .expect("Failed to send request."); // This should yield a 302 redirect with some query params. - assert_eq!(response.status(), reqwest::StatusCode::FOUND); + assert_eq!(response.status(), StatusCode::FOUND); assert_no_cache!(response); // And we should have a URL in the location header. @@ -813,7 +814,7 @@ async fn test_oauth2_openid_public_flow_impl( .await .expect("Failed to send code exchange request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); assert_no_cache!(response); // The body is a json AccessTokenResponse @@ -858,10 +859,10 @@ async fn test_oauth2_openid_public_flow_impl( .await .expect("Failed to send userinfo preflight request."); - assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.status(), StatusCode::OK); let cors_header: &str = response .headers() - .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN) + .get(header::ACCESS_CONTROL_ALLOW_ORIGIN) .expect("missing access-control-allow-origin header") .to_str() .expect("invalid access-control-allow-origin header"); @@ -931,6 +932,7 @@ async fn test_oauth2_token_post_bad_bodies(rsclient: KanidmClient) { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) + .tls_built_in_native_certs(false) .no_proxy() .build() .expect("Failed to create client."); @@ -966,6 +968,7 @@ async fn test_oauth2_token_revoke_post(rsclient: KanidmClient) { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) + .tls_built_in_native_certs(false) .no_proxy() .build() .expect("Failed to create client."); diff --git a/server/testkit/tests/person.rs b/server/testkit/tests/testkit/person.rs similarity index 93% rename from server/testkit/tests/person.rs rename to server/testkit/tests/testkit/person.rs index e8ce8f4fe..8e4bf95c9 100644 --- a/server/testkit/tests/person.rs +++ b/server/testkit/tests/testkit/person.rs @@ -1,4 +1,4 @@ -use kanidm_client::{ClientError, KanidmClient}; +use kanidm_client::{ClientError, KanidmClient, StatusCode}; use kanidm_proto::constants::ATTR_MAIL; use kanidmd_testkit::{create_user, ADMIN_TEST_PASSWORD, ADMIN_TEST_USER}; use serde_json::Value; @@ -48,6 +48,6 @@ async fn test_v1_person_id_ssh_pubkeys_post(rsclient: KanidmClient) { eprintln!("response: {:#?}", response); assert!(matches!( response, - ClientError::Http(reqwest::StatusCode::BAD_REQUEST, _, _) + ClientError::Http(StatusCode::BAD_REQUEST, _, _) )); } diff --git a/server/testkit/tests/proto_v1_test.rs b/server/testkit/tests/testkit/proto_v1_test.rs similarity index 99% rename from server/testkit/tests/proto_v1_test.rs rename to server/testkit/tests/testkit/proto_v1_test.rs index 577d14d30..48ea88706 100644 --- a/server/testkit/tests/proto_v1_test.rs +++ b/server/testkit/tests/testkit/proto_v1_test.rs @@ -942,9 +942,8 @@ async fn test_server_rest_oauth2_basic_lifecycle(rsclient: KanidmClient) { assert!(res.is_ok()); //test getting the image - let client = reqwest::Client::new(); - - let response = client + let response = rsclient + .client() .get(rsclient.make_url("/ui/images/oauth2/test_integration")) .bearer_auth(rsclient.get_token().await.unwrap()); @@ -1780,7 +1779,8 @@ async fn start_password_session( password: &str, privileged: bool, ) -> Result<UserAuthToken, ()> { - let client = reqwest::Client::new(); + let fresh_rsclient = rsclient.new_session().unwrap(); + let client = fresh_rsclient.client(); let authreq = AuthRequest { step: AuthStep::Init2 { diff --git a/server/testkit/tests/scim_test.rs b/server/testkit/tests/testkit/scim_test.rs similarity index 74% rename from server/testkit/tests/scim_test.rs rename to server/testkit/tests/testkit/scim_test.rs index 674e546b5..8e947790f 100644 --- a/server/testkit/tests/scim_test.rs +++ b/server/testkit/tests/testkit/scim_test.rs @@ -5,7 +5,6 @@ use kanidm_proto::scim_v1::ScimEntryGetQuery; use kanidmd_lib::constants::NAME_IDM_ADMINS; use kanidmd_lib::prelude::Attribute; use kanidmd_testkit::{ADMIN_TEST_PASSWORD, ADMIN_TEST_USER}; -use reqwest::header::HeaderValue; use std::str::FromStr; use url::Url; @@ -104,64 +103,6 @@ async fn test_sync_account_lifecycle(rsclient: KanidmClient) { .expect("Failed to destroy token"); } -#[kanidmd_testkit::test] -async fn test_scim_sync_get(rsclient: KanidmClient) { - // We need to do manual reqwests here. - - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert( - reqwest::header::AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {:?}", rsclient.get_token().await)).unwrap(), - ); - - let client = reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .default_headers(headers) - .build() - .unwrap(); - - // here we test the /ui/ endpoint which should have the headers - let response = match client.get(rsclient.make_url("/scim/v1/Sync")).send().await { - Ok(value) => value, - Err(error) => { - panic!( - "Failed to query {:?} : {:#?}", - rsclient.make_url("/scim/v1/Sync"), - error - ); - } - }; - eprintln!("response: {:#?}", response); - assert!(response.status().is_client_error()); - - // check that the CSP headers are coming back - eprintln!( - "csp headers: {:#?}", - response - .headers() - .get(http::header::CONTENT_SECURITY_POLICY) - ); - assert_ne!( - response - .headers() - .get(http::header::CONTENT_SECURITY_POLICY), - None - ); - - // test that the proper content type comes back - let url = rsclient.make_url("/scim/v1/Sink"); - let response = match client.get(url.clone()).send().await { - Ok(value) => value, - Err(error) => { - panic!("Failed to query {:?} : {:#?}", url, error); - } - }; - assert!(response.status().is_success()); - let content_type = response.headers().get(http::header::CONTENT_TYPE).unwrap(); - assert!(content_type.to_str().unwrap().contains("text/html")); - assert!(response.text().await.unwrap().contains("Sink")); -} - #[kanidmd_testkit::test] async fn test_scim_sync_entry_get(rsclient: KanidmClient) { let res = rsclient diff --git a/server/testkit/tests/service_account.rs b/server/testkit/tests/testkit/service_account.rs similarity index 88% rename from server/testkit/tests/service_account.rs rename to server/testkit/tests/testkit/service_account.rs index be0dcaed5..1fed6b3aa 100644 --- a/server/testkit/tests/service_account.rs +++ b/server/testkit/tests/testkit/service_account.rs @@ -4,10 +4,7 @@ use kanidm_client::KanidmClient; #[kanidmd_testkit::test] async fn test_v1_service_account_id_attr_attr_delete(rsclient: KanidmClient) { // We need to do manual reqwests here. - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); // let post_body = serde_json::json!({"filter": "self"}).to_string(); diff --git a/server/testkit/tests/system.rs b/server/testkit/tests/testkit/system.rs similarity index 86% rename from server/testkit/tests/system.rs rename to server/testkit/tests/testkit/system.rs index 6c5c22095..d0d7f46c3 100644 --- a/server/testkit/tests/system.rs +++ b/server/testkit/tests/testkit/system.rs @@ -3,10 +3,7 @@ use kanidm_client::KanidmClient; /// This literally tests that the thing exists and responds in a way we expect, probably worth testing it better... #[kanidmd_testkit::test] async fn test_v1_system_post_attr(rsclient: KanidmClient) { - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = rsclient.client(); let response = match client .post(rsclient.make_url("/v1/system/_attr/domain_name")) diff --git a/server/testkit/tests/unix.rs b/server/testkit/tests/testkit/unix.rs similarity index 100% rename from server/testkit/tests/unix.rs rename to server/testkit/tests/testkit/unix.rs diff --git a/tools/cli/Cargo.toml b/tools/cli/Cargo.toml index 219a8ed82..fdb9e2dfd 100644 --- a/tools/cli/Cargo.toml +++ b/tools/cli/Cargo.toml @@ -22,14 +22,14 @@ dev-oauth2-device-flow = [] [lib] name = "kanidm_cli" path = "src/cli/lib.rs" -test = true +test = false doctest = false [[bin]] name = "kanidm" path = "src/cli/main.rs" doc = false -test = true +test = false doctest = false [[bin]] diff --git a/tools/device_flow/Cargo.toml b/tools/device_flow/Cargo.toml index 1ac671ba8..252353711 100644 --- a/tools/device_flow/Cargo.toml +++ b/tools/device_flow/Cargo.toml @@ -12,8 +12,8 @@ repository = { workspace = true } [lib] -test = false doctest = false +test = false [features] diff --git a/tools/iam_migrations/freeipa/Cargo.toml b/tools/iam_migrations/freeipa/Cargo.toml index 6eee6a97a..4b79a812c 100644 --- a/tools/iam_migrations/freeipa/Cargo.toml +++ b/tools/iam_migrations/freeipa/Cargo.toml @@ -11,6 +11,11 @@ license = { workspace = true } homepage = { workspace = true } repository = { workspace = true } +[[bin]] +name = "kanidm-ipa-sync" +test = false +doctest = false + [dependencies] clap = { workspace = true, features = ["derive", "env"] } chrono = { workspace = true } diff --git a/tools/iam_migrations/ldap/Cargo.toml b/tools/iam_migrations/ldap/Cargo.toml index e8d240d03..8ebed8d00 100644 --- a/tools/iam_migrations/ldap/Cargo.toml +++ b/tools/iam_migrations/ldap/Cargo.toml @@ -11,6 +11,11 @@ license = { workspace = true } homepage = { workspace = true } repository = { workspace = true } +[[bin]] +name = "kanidm-ldap-sync" +test = false +doctest = false + [dependencies] clap = { workspace = true, features = ["derive", "env"] } chrono = { workspace = true } diff --git a/tools/orca/Cargo.toml b/tools/orca/Cargo.toml index 08aa6dfbc..5315f0212 100644 --- a/tools/orca/Cargo.toml +++ b/tools/orca/Cargo.toml @@ -14,7 +14,7 @@ repository = { workspace = true } [[bin]] name = "orca" path = "src/main.rs" -test = true +test = false doctest = false [dependencies] diff --git a/unix_integration/resolver/Cargo.toml b/unix_integration/resolver/Cargo.toml index 8718d9c53..96e09bbee 100644 --- a/unix_integration/resolver/Cargo.toml +++ b/unix_integration/resolver/Cargo.toml @@ -21,28 +21,28 @@ tpm = ["kanidm-hsm-crypto/tpm"] name = "kanidm_unixd" path = "src/bin/kanidm_unixd.rs" required-features = ["unix"] -test = true +test = false doctest = false [[bin]] name = "kanidm_unixd_tasks" path = "src/bin/kanidm_unixd_tasks.rs" required-features = ["unix"] -test = true +test = false doctest = false [[bin]] name = "kanidm_ssh_authorizedkeys" path = "src/bin/kanidm_ssh_authorizedkeys.rs" required-features = ["unix"] -test = true +test = false doctest = false [[bin]] name = "kanidm-unix" path = "src/bin/kanidm-unix.rs" required-features = ["unix"] -test = true +test = false doctest = false [lib] diff --git a/unix_integration/resolver/tests/cache_layer_test.rs b/unix_integration/resolver/tests/cache_layer_test.rs index 76bbb7cda..7e5b9c0fe 100644 --- a/unix_integration/resolver/tests/cache_layer_test.rs +++ b/unix_integration/resolver/tests/cache_layer_test.rs @@ -88,6 +88,7 @@ async fn setup_test(fix_fn: Fixture) -> (Resolver, KanidmClient) { // Run fixtures let adminclient = KanidmClientBuilder::new() .address(addr.clone()) + .enable_native_ca_roots(false) .no_proxy() .build() .expect("Failed to build sync client"); @@ -96,12 +97,14 @@ async fn setup_test(fix_fn: Fixture) -> (Resolver, KanidmClient) { let client = KanidmClientBuilder::new() .address(addr.clone()) + .enable_native_ca_roots(false) .no_proxy() .build() .expect("Failed to build async admin client"); let rsclient = KanidmClientBuilder::new() .address(addr) + .enable_native_ca_roots(false) .no_proxy() .build() .expect("Failed to build client");