68 20230821 replication (#2020)

* Resolve spn incremental replication

Cargo.lock

@@ -207,6 +207,28 @@ dependencies = [
  "syn 2.0.29",
 ]
 
+[[package]]
+name = "async-stream"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.29",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.73"
@@ -415,6 +437,16 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "base64urlsafedata"
+version = "0.1.3"
+source = "git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5#429662e34d6e760af8cff68760567c6b56dbb2d5"
+dependencies = [
+ "base64 0.21.2",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "bincode"
 version = "1.3.3"
@@ -715,7 +747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51f9032b96a89dd79ffc5f62523d5351ebb40680cbdfc4029393b511b9e971aa"
 dependencies = [
  "base64 0.13.1",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex",
  "openssl",
  "serde",
@@ -1250,7 +1282,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.28",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -2166,7 +2198,7 @@ dependencies = [
 name = "kanidm-ipa-sync"
 version = "1.1.0-rc.14-dev"
 dependencies = [
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono",
  "clap",
  "clap_complete",
@@ -2190,7 +2222,7 @@ dependencies = [
 name = "kanidm-ldap-sync"
 version = "1.1.0-rc.14-dev"
 dependencies = [
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono",
  "clap",
  "clap_complete",
@@ -2243,7 +2275,7 @@ version = "0.1.0"
 dependencies = [
  "argon2",
  "base64 0.21.2",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex",
  "kanidm_proto",
  "openssl",
@@ -2268,7 +2300,7 @@ name = "kanidm_proto"
 version = "1.1.0-rc.14-dev"
 dependencies = [
  "base32",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_enum",
  "scim_proto",
  "serde",
@@ -2316,7 +2348,7 @@ name = "kanidm_unix_int"
 version = "1.1.0-rc.14-dev"
 dependencies = [
  "async-trait",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes",
  "clap",
  "clap_complete",
@@ -2405,7 +2437,7 @@ name = "kanidmd_lib"
 version = "1.1.0-rc.14-dev"
 dependencies = [
  "base64 0.21.2",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "compact_jwt",
  "concread",
  "criterion",
@@ -2567,7 +2599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a229cd5ee2a4e5a1a279b6216494aa2a5053a189c5ce37bb31f9156b63b63de"
 dependencies = [
  "base64 0.13.1",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-util",
  "ldap3_proto",
  "openssl",
@@ -3872,7 +3904,7 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38e53f2c444b72dd7410aa1cdc3c0942349262e84364dc7968dc7402525ea2ca"
 dependencies = [
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "peg",
  "serde",
  "serde_json",
@@ -4529,6 +4561,7 @@ dependencies = [
  "futures-core",
  "pin-project-lite",
  "tokio",
+ "tokio-util",
 ]
 
 [[package]]
@@ -5027,32 +5060,42 @@ dependencies = [
 
 [[package]]
 name = "webauthn-authenticator-rs"
-version = "0.4.9"
+version = "0.5.0-dev"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5#429662e34d6e760af8cff68760567c6b56dbb2d5"
-checksum = "603b8602cae2d6c3706b6195765ff582389494d10c442d84a1de2ed5a25679ef"
 dependencies = [
+ "async-stream",
+ "async-trait",
  "authenticator-ctap2-2021",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5)",
+ "bitflags 1.3.2",
+ "futures",
+ "hex",
  "nom",
+ "num-derive",
+ "num-traits",
  "openssl",
  "rpassword 5.0.1",
  "serde",
+ "serde_bytes",
  "serde_cbor_2",
  "serde_json",
+ "tokio",
+ "tokio-stream",
  "tracing",
+ "unicode-normalization",
  "url",
  "uuid",
+ "webauthn-rs-core",
  "webauthn-rs-proto",
  "windows 0.41.0",
 ]
 
 [[package]]
 name = "webauthn-rs"
-version = "0.4.8"
+version = "0.5.0-dev"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5#429662e34d6e760af8cff68760567c6b56dbb2d5"
-checksum = "2db00711c712414e93b019c4596315085792215bc2ac2d5872f9e8913b0a6316"
 dependencies = [
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5)",
  "serde",
  "tracing",
  "url",
@@ -5062,12 +5105,11 @@ dependencies = [
 
 [[package]]
 name = "webauthn-rs-core"
-version = "0.4.9"
+version = "0.5.0-dev"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5#429662e34d6e760af8cff68760567c6b56dbb2d5"
-checksum = "294c78c83f12153a51e1cf1e6970b5da1397645dada39033a9c3173a8fc4fc2b"
 dependencies = [
- "base64 0.13.1",
+ "base64 0.21.2",
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5)",
  "compact_jwt",
  "der-parser",
  "nom",
@@ -5086,11 +5128,10 @@ dependencies = [
 
 [[package]]
 name = "webauthn-rs-proto"
-version = "0.4.9"
+version = "0.5.0-dev"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5#429662e34d6e760af8cff68760567c6b56dbb2d5"
-checksum = "d24e638361a63ba5c0a0be6a60229490fcdf33740ed63df5bb6bdb627b52a138"
 dependencies = [
- "base64urlsafedata",
+ "base64urlsafedata 0.1.3 (git+https://github.com/kanidm/webauthn-rs.git?rev=429662e34d6e760af8cff68760567c6b56dbb2d5)",
  "js-sys",
  "serde",
  "serde-wasm-bindgen 0.4.5",

Cargo.toml

@@ -165,8 +165,8 @@ tokio-util = "^0.7.8"
 
 toml = "^0.5.11"
 touch = "^0.0.1"
-# tracing = { version = "^0.1.37", features = ["max_level_trace", "release_max_level_debug"] }
+tracing = { version = "^0.1.37", features = ["max_level_trace", "release_max_level_debug"] }
-tracing = { version = "^0.1.37" }
+# tracing = { version = "^0.1.37" }
 tracing-subscriber = { version = "^0.3.17", features = ["env-filter"] }
 
 # tracing-forest = { path = "/Users/william/development/tracing-forest/tracing-forest" }
@@ -183,14 +183,20 @@ wasm-bindgen = "^0.2.86"
 wasm-bindgen-futures = "^0.4.30"
 wasm-bindgen-test = "0.3.35"
 
-webauthn-authenticator-rs = "0.4.8"
+# webauthn-authenticator-rs = { version = "0.4.8", features = ["softpasskey", "softtoken"] }
-webauthn-rs = "0.4.8"
+# webauthn-rs = "0.4.8"
-webauthn-rs-core = "0.4.8"
+# webauthn-rs-core = "0.4.8"
-webauthn-rs-proto = "0.4.8"
+# webauthn-rs-proto = "0.4.8"
-# webauthn-authenticator-rs = { path = "../webauthn-rs/webauthn-authenticator-rs" }
+# webauthn-authenticator-rs = { path = "../webauthn-rs/webauthn-authenticator-rs", features = ["softpasskey", "softtoken", "mozilla"] }
-# webauthn-rs = { path = "../webauthn-rs/webauthn-rs" }
+# webauthn-rs = { path = "../webauthn-rs/webauthn-rs", features = ["preview-features"] }
 # webauthn-rs-core = { path = "../webauthn-rs/webauthn-rs-core" }
 # webauthn-rs-proto = { path = "../webauthn-rs/webauthn-rs-proto" }
+
+webauthn-authenticator-rs = { git = "https://github.com/kanidm/webauthn-rs.git", rev = "429662e34d6e760af8cff68760567c6b56dbb2d5", features = ["softpasskey", "softtoken", "mozilla"] }
+webauthn-rs = { git = "https://github.com/kanidm/webauthn-rs.git", rev = "429662e34d6e760af8cff68760567c6b56dbb2d5", features = ["preview-features"] }
+webauthn-rs-core = { git = "https://github.com/kanidm/webauthn-rs.git", rev = "429662e34d6e760af8cff68760567c6b56dbb2d5" }
+webauthn-rs-proto = { git = "https://github.com/kanidm/webauthn-rs.git", rev = "429662e34d6e760af8cff68760567c6b56dbb2d5" }
+
 web-sys = "^0.3.62"
 whoami = "^1.4.1"
 walkdir = "2"

server/lib/src/be/dbvalue.rs

@@ -7,7 +7,7 @@ use serde_with::skip_serializing_none;
 use url::Url;
 use uuid::Uuid;
 use webauthn_rs::prelude::{
-    DeviceKey as DeviceKeyV4, Passkey as PasskeyV4, SecurityKey as SecurityKeyV4,
+    AttestedPasskey as DeviceKeyV4, Passkey as PasskeyV4, SecurityKey as SecurityKeyV4,
 };
 use webauthn_rs_core::proto::{COSEKey, UserVerificationPolicy};
 

server/lib/src/be/idl_arc_sqlite.rs

@@ -1086,9 +1086,12 @@ impl<'a> IdlArcSqliteWriteTransaction<'a> {
     ///
     /// It should only be called internally by the backend in limited and
     /// specific situations.
+    #[instrument(level = "trace", skip_all)]
     pub fn danger_purge_idxs(&mut self) -> Result<(), OperationError> {
+        error!("CLEARING CACHE");
         self.db.danger_purge_idxs().map(|()| {
             self.idl_cache.clear();
+            self.name_cache.clear();
         })
     }
 
@@ -1096,6 +1099,7 @@ impl<'a> IdlArcSqliteWriteTransaction<'a> {
     ///
     /// It should only be called internally by the backend in limited and
     /// specific situations.
+    #[instrument(level = "trace", skip_all)]
     pub fn danger_purge_id2entry(&mut self) -> Result<(), OperationError> {
         self.db.danger_purge_id2entry().map(|()| {
             let mut ids = IDLBitRange::new();

server/lib/src/be/idl_sqlite.rs

@@ -348,7 +348,10 @@ pub trait IdlSqliteTransaction {
         // The table exists - lets now get the actual index itself.
         let mut stmt = self
             .get_conn()?
-            .prepare("SELECT rdn FROM idx_uuid2rdn WHERE uuid = :uuid")
+            .prepare(&format!(
+                "SELECT rdn FROM {}.idx_uuid2rdn WHERE uuid = :uuid",
+                self.get_db_name()
+            ))
             .map_err(sqlite_error)?;
         let rdn: Option<String> = stmt
             .query_row(&[(":uuid", &uuids)], |row| row.get(0))
@@ -363,7 +366,14 @@ pub trait IdlSqliteTransaction {
         // Try to get a value.
         let data: Option<Vec<u8>> = self
             .get_conn()?
-            .query_row("SELECT data FROM db_sid WHERE id = 2", [], |row| row.get(0))
+            .query_row(
+                &format!(
+                    "SELECT data FROM {}.db_sid WHERE id = 2",
+                    self.get_db_name()
+                ),
+                [],
+                |row| row.get(0),
+            )
             .optional()
             // this whole map call is useless
             .map(|e_opt| {
@@ -394,7 +404,14 @@ pub trait IdlSqliteTransaction {
         // Try to get a value.
         let data: Option<Vec<u8>> = self
             .get_conn()?
-            .query_row("SELECT data FROM db_did WHERE id = 2", [], |row| row.get(0))
+            .query_row(
+                &format!(
+                    "SELECT data FROM {}.db_did WHERE id = 2",
+                    self.get_db_name()
+                ),
+                [],
+                |row| row.get(0),
+            )
             .optional()
             // this whole map call is useless
             .map(|e_opt| {
@@ -425,9 +442,14 @@ pub trait IdlSqliteTransaction {
         // Try to get a value.
         let data: Option<Vec<u8>> = self
             .get_conn()?
-            .query_row("SELECT data FROM db_op_ts WHERE id = 1", [], |row| {
+            .query_row(
-                row.get(0)
+                &format!(
-            })
+                    "SELECT data FROM {}.db_op_ts WHERE id = 1",
+                    self.get_db_name()
+                ),
+                [],
+                |row| row.get(0),
+            )
             .optional()
             .map(|e_opt| {
                 // If we have a row, we try to make it a sid
@@ -457,7 +479,7 @@ pub trait IdlSqliteTransaction {
     fn get_allids(&self) -> Result<IDLBitRange, OperationError> {
         let mut stmt = self
             .get_conn()?
-            .prepare("SELECT id FROM id2entry")
+            .prepare(&format!("SELECT id FROM {}.id2entry", self.get_db_name()))
             .map_err(sqlite_error)?;
         let res = stmt.query_map([], |row| row.get(0)).map_err(sqlite_error)?;
         let mut ids: Result<IDLBitRange, _> = res
@@ -480,7 +502,10 @@ pub trait IdlSqliteTransaction {
     fn list_idxs(&self) -> Result<Vec<String>, OperationError> {
         let mut stmt = self
             .get_conn()?
-            .prepare("SELECT name from sqlite_master where type='table' and name GLOB 'idx_*'")
+            .prepare(&format!(
+                "SELECT name from {}.sqlite_master where type='table' and name GLOB 'idx_*'",
+                self.get_db_name()
+            ))
             .map_err(sqlite_error)?;
         let idx_table_iter = stmt.query_map([], |row| row.get(0)).map_err(sqlite_error)?;
 
@@ -547,7 +572,7 @@ pub trait IdlSqliteTransaction {
         // TODO: Once we have slopes we can add .exists_table, and assert
         // it's an idx table.
 
-        let query = format!("SELECT key, idl FROM {index_name}");
+        let query = format!("SELECT key, idl FROM {}.{}", self.get_db_name(), index_name);
         let mut stmt = self
             .get_conn()?
             .prepare(query.as_str())
@@ -1070,11 +1095,13 @@ impl IdlSqliteWriteTransaction {
     ///
     /// It should only be called internally by the backend in limited and
     /// specific situations.
+    #[instrument(level = "trace", skip_all)]
     pub fn danger_purge_idxs(&self) -> Result<(), OperationError> {
         let idx_table_list = self.list_idxs()?;
+        trace!(tables = ?idx_table_list);
 
         idx_table_list.iter().try_for_each(|idx_table| {
-            trace!(table = ?idx_table, "removing idx_table");
+            debug!(table = ?idx_table, "removing idx_table");
             self.get_conn()?
                 .prepare(format!("DROP TABLE {}.{}", self.get_db_name(), idx_table).as_str())
                 .and_then(|mut stmt| stmt.execute([]).map(|_| ()))
@@ -1238,6 +1265,7 @@ impl IdlSqliteWriteTransaction {
     ///
     /// It should only be called internally by the backend in limited and
     /// specific situations.
+    #[instrument(level = "trace", skip_all)]
     pub fn danger_purge_id2entry(&self) -> Result<(), OperationError> {
         self.get_conn()?
             .execute(&format!("DELETE FROM {}.id2entry", self.get_db_name()), [])

server/lib/src/be/mod.rs

@@ -1162,7 +1162,7 @@ impl<'a> BackendWriteTransaction<'a> {
                 }
                 Some(idl) => {
                     // BUG - duplicate uuid!
-                    error!(uuid = ?ctx_ent_uuid, "Invalid IDL state, uuid index must have only a single or no values. Contains {}", idl.len());
+                    error!(uuid = ?ctx_ent_uuid, "Invalid IDL state, uuid index must have only a single or no values. Contains {:?}", idl);
                     return Err(OperationError::InvalidDbState);
                 }
                 None => {
@@ -1491,6 +1491,11 @@ impl<'a> BackendWriteTransaction<'a> {
                         match self.idlayer.get_idl(attr, itype, &idx_key)? {
                             Some(mut idl) => {
                                 idl.insert_id(e_id);
+                                if cfg!(debug_assertions)
+                                    && attr == "uuid" && itype == IndexType::Equality {
+                                        trace!("{:?}", idl);
+                                        debug_assert!(idl.len() <= 1);
+                                }
                                 self.idlayer.write_idl(attr, itype, &idx_key, &idl)
                             }
                             None => {
@@ -1507,6 +1512,10 @@ impl<'a> BackendWriteTransaction<'a> {
                         match self.idlayer.get_idl(attr, itype, &idx_key)? {
                             Some(mut idl) => {
                                 idl.remove_id(e_id);
+                                if cfg!(debug_assertions) && attr == "uuid" && itype == IndexType::Equality {
+                                        trace!("{:?}", idl);
+                                        debug_assert!(idl.len() <= 1);
+                                }
                                 self.idlayer.write_idl(attr, itype, &idx_key, &idl)
                             }
                             None => {

server/lib/src/entry.rs

@@ -42,7 +42,7 @@ use smartstring::alias::String as AttrString;
 use time::OffsetDateTime;
 use tracing::trace;
 use uuid::Uuid;
-use webauthn_rs::prelude::{DeviceKey as DeviceKeyV4, Passkey as PasskeyV4};
+use webauthn_rs::prelude::{AttestedPasskey as DeviceKeyV4, Passkey as PasskeyV4};
 
 use crate::be::dbentry::{DbEntry, DbEntryV2, DbEntryVers};
 use crate::be::dbvalue::DbValueSetV2;

server/lib/src/idm/account.rs

@@ -7,7 +7,7 @@ use kanidm_proto::v1::{
 use time::OffsetDateTime;
 use uuid::Uuid;
 use webauthn_rs::prelude::{
-    AuthenticationResult, CredentialID, DeviceKey as DeviceKeyV4, Passkey as PasskeyV4,
+    AttestedPasskey as DeviceKeyV4, AuthenticationResult, CredentialID, Passkey as PasskeyV4,
 };
 
 use crate::constants::UUID_ANONYMOUS;

server/lib/src/idm/authsession.rs

@@ -15,10 +15,9 @@ use kanidm_proto::v1::{
     AuthAllowed, AuthCredential, AuthIssueSession, AuthMech, OperationError, UserAuthToken,
 };
 // use crossbeam::channel::Sender;
+use nonempty::{nonempty, NonEmpty};
 use tokio::sync::mpsc::UnboundedSender as Sender;
 use uuid::Uuid;
-// use webauthn_rs::prelude::DeviceKey as DeviceKeyV4;
-use nonempty::{nonempty, NonEmpty};
 use webauthn_rs::prelude::Passkey as PasskeyV4;
 use webauthn_rs::prelude::{
     CredentialID, PasskeyAuthentication, RequestChallengeResponse, SecurityKeyAuthentication,
@@ -1833,7 +1832,7 @@ mod tests {
     ) {
         let webauthn = create_webauthn();
         // Setup a soft token
-        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new());
+        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new(true));
 
         let uuid = Uuid::new_v4();
 
@@ -1861,7 +1860,7 @@ mod tests {
     ) {
         let webauthn = create_webauthn();
         // Setup a soft token
-        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new());
+        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new(true));
 
         let uuid = Uuid::new_v4();
 
@@ -1986,7 +1985,7 @@ mod tests {
 
         // Use an incorrect softtoken.
         {
-            let mut inv_wa = WebauthnAuthenticator::new(SoftPasskey::new());
+            let mut inv_wa = WebauthnAuthenticator::new(SoftPasskey::new(true));
             let (chal, reg_state) = webauthn
                 .start_passkey_registration(account.uuid, &account.name, &account.displayname, None)
                 .expect("Failed to setup webauthn rego challenge");

server/lib/src/idm/credupdatesession.rs

@@ -12,8 +12,8 @@ use kanidm_proto::v1::{
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use webauthn_rs::prelude::{
-    CreationChallengeResponse, DeviceKey as DeviceKeyV4, Passkey as PasskeyV4, PasskeyRegistration,
+    AttestedPasskey as DeviceKeyV4, CreationChallengeResponse, Passkey as PasskeyV4,
-    RegisterPublicKeyCredential,
+    PasskeyRegistration, RegisterPublicKeyCredential,
 };
 
 use crate::credential::totp::{Totp, TOTP_DEFAULT_STEP};
@@ -2679,7 +2679,7 @@ mod tests {
         let origin = cutxn.get_origin().clone();
 
         // Create a soft passkey
-        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new());
+        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new(true));
 
         // Start the registration
         let c_status = cutxn

server/lib/src/idm/reauth.rs

@@ -230,7 +230,7 @@ mod tests {
         let cutxn = idms.cred_update_transaction().await;
         let origin = cutxn.get_origin().clone();
 
-        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new());
+        let mut wa = WebauthnAuthenticator::new(SoftPasskey::new(true));
 
         let c_status = cutxn
             .credential_passkey_init(&cust, ct)

server/lib/src/idm/server.rs

@@ -2072,6 +2072,13 @@ impl<'a> IdmServerProxyWriteTransaction<'a> {
                 .map(|new_cookie_key| {
                     self.domain_keys.cookie_key = new_cookie_key;
                 })?;
+            // If the domain name has changed, we need to update rp-id in
+            // webauthn rs
+            //
+            // TODO: I'm not sure actually. because on a domain rename we
+            // might need to update origin too. So this gets a bit tricky.
+            // we might actually need to *not* reload here, and then let the
+            // admin do it inline with their configs too.
         }
         // Commit everything.
         self.oauth2rs.commit();

server/lib/src/plugins/memberof.rs

@@ -230,6 +230,18 @@ impl Plugin for MemberOf {
         Self::post_create_inner(qs, cand, &ident)
     }
 
+    #[instrument(level = "debug", name = "memberof_post_repl_incremental", skip_all)]
+    fn post_repl_incremental(
+        qs: &mut QueryServerWriteTransaction,
+        pre_cand: &[Arc<EntrySealedCommitted>],
+        cand: &[EntrySealedCommitted],
+    ) -> Result<(), OperationError> {
+        // IMPORTANT - we need this for now so that dyngroup doesn't error on us, since
+        // repl is internal and dyngroup has a safety check to prevent external triggers.
+        let ident_internal = Identity::from_internal();
+        Self::post_modify_inner(qs, pre_cand, cand, &ident_internal)
+    }
+
     #[instrument(level = "debug", name = "memberof_post_modify", skip_all)]
     fn post_modify(
         qs: &mut QueryServerWriteTransaction,

server/lib/src/plugins/spn.rs

@@ -73,6 +73,14 @@ impl Plugin for Spn {
         Self::post_modify_inner(qs, pre_cand, cand)
     }
 
+    fn post_repl_incremental(
+        qs: &mut QueryServerWriteTransaction,
+        pre_cand: &[Arc<EntrySealedCommitted>],
+        cand: &[EntrySealedCommitted],
+    ) -> Result<(), OperationError> {
+        Self::post_modify_inner(qs, pre_cand, cand)
+    }
+
     #[instrument(level = "debug", name = "spn::verify", skip_all)]
     fn verify(qs: &mut QueryServerReadTransaction) -> Vec<Result<(), ConsistencyError>> {
         // Verify that all items with spn's have valid spns.
@@ -164,9 +172,7 @@ impl Spn {
         pre_cand: &[Arc<Entry<EntrySealed, EntryCommitted>>],
         cand: &[Entry<EntrySealed, EntryCommitted>],
     ) -> Result<(), OperationError> {
-        // On modify, if changing domain_name on UUID_DOMAIN_INFO
+        // On modify, if changing domain_name on UUID_DOMAIN_INFO trigger the spn regen
-        //    trigger the spn regen ... which is expensive. Future
-        // TODO #157: will be improvements to modify on large txns.
 
         let domain_name_changed = cand.iter().zip(pre_cand.iter()).find_map(|(post, pre)| {
             let domain_name = post.get_ava_single("domain_name");
@@ -183,6 +189,12 @@ impl Spn {
             return Ok(())
         };
 
+        // IMPORTANT - we have to *pre-emptively reload the domain info here*
+        //
+        // If we don't, we don't get the updated domain name in the txn, and then
+        // spn rename fails as we recurse and just populate the old name.
+        qs.reload_domain_info()?;
+
         admin_info!(
             "IMPORTANT!!! Changing domain name to \"{:?}\". THIS MAY TAKE A LONG TIME ...",
             domain_name

server/lib/src/repl/consumer.rs

@@ -127,13 +127,6 @@ impl<'a> QueryServerWriteTransaction<'a> {
                 (sealed_ent, db_ent)
             })
             .collect::<Vec<_>>();
-        /*
-        .collect::<Result<Vec<_>, _>>()
-        .map_err(|e| {
-            error!(err = ?e, "Failed to validate schema of incremental entries");
-            OperationError::SchemaViolation(e)
-        })?;
-        */
 
         // We now have three sets!
         //
@@ -198,12 +191,21 @@ impl<'a> QueryServerWriteTransaction<'a> {
                     )
                 });
         }
+        if !self.changed_sync_agreement {
+            self.changed_sync_agreement = cand
+                .iter()
+                .chain(pre_cand.iter().map(|e| e.as_ref()))
+                .any(|e| {
+                    e.attribute_equality(Attribute::Class.as_ref(), &EntryClass::SyncAccount.into())
+                });
+        }
 
         trace!(
             schema_reload = ?self.changed_schema,
             acp_reload = ?self.changed_acp,
             oauth2_reload = ?self.changed_oauth2,
             domain_reload = ?self.changed_domain,
+            changed_sync_agreement = ?self.changed_sync_agreement
         );
 
         Ok(())

server/lib/src/repl/proto.rs

@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::{BTreeMap, BTreeSet};
 
 use webauthn_rs::prelude::{
-    DeviceKey as DeviceKeyV4, Passkey as PasskeyV4, SecurityKey as SecurityKeyV4,
+    AttestedPasskey as DeviceKeyV4, Passkey as PasskeyV4, SecurityKey as SecurityKeyV4,
 };
 
 // Re-export this for our own usage.

server/lib/src/repl/tests.rs

@@ -1774,6 +1774,165 @@ async fn test_repl_increment_consumer_lagging_attributes(
 }
 
 // Test change of a domain name over incremental.
+#[qs_pair_test]
+async fn test_repl_increment_domain_rename(server_a: &QueryServer, server_b: &QueryServer) {
+    let ct = duration_from_epoch_now();
+
+    let mut server_a_txn = server_a.write(ct).await;
+    let mut server_b_txn = server_b.read().await;
+
+    assert!(repl_initialise(&mut server_b_txn, &mut server_a_txn)
+        .and_then(|_| server_a_txn.commit())
+        .is_ok());
+    drop(server_b_txn);
+
+    // Rename the domain. We do this on server_a.
+    let mut server_a_txn = server_a.write(ct).await;
+    assert!(server_a_txn
+        .danger_domain_rename("new.example.com")
+        .and_then(|_| server_a_txn.commit())
+        .is_ok());
+
+    // Add an entry to server_b. This should have it's spn regenerated after
+    // the domain rename is replicated.
+    // - satifies:
+    // Test domain rename where the reciever of the rename has added entries, and
+    // they need spn regen to stabilise.
+
+    let mut server_b_txn = server_b.write(duration_from_epoch_now()).await;
+    let t_uuid = Uuid::new_v4();
+    assert!(server_b_txn
+        .internal_create(vec![entry_init!(
+            (Attribute::Class.as_ref(), EntryClass::Object.to_value()),
+            (Attribute::Class.as_ref(), EntryClass::Account.to_value()),
+            (Attribute::Class.as_ref(), EntryClass::Person.to_value()),
+            (Attribute::Name.as_ref(), Value::new_iname("testperson1")),
+            (Attribute::Uuid.as_ref(), Value::Uuid(t_uuid)),
+            (
+                Attribute::Description.as_ref(),
+                Value::new_utf8s("testperson1")
+            ),
+            (
+                Attribute::DisplayName.as_ref(),
+                Value::new_utf8s("testperson1")
+            )
+        ),])
+        .is_ok());
+    server_b_txn.commit().expect("Failed to commit");
+
+    // Now replicate from a to b. This will be fun won't it.
+    // This means A -> B - no change on B, it's the persisting tombstone.
+    let mut server_a_txn = server_a.read().await;
+    let mut server_b_txn = server_b.write(duration_from_epoch_now()).await;
+
+    trace!("========================================");
+    repl_incremental(&mut server_a_txn, &mut server_b_txn);
+
+    let e1 = server_a_txn
+        .internal_search_all_uuid(UUID_DOMAIN_INFO)
+        .expect("Unable to access new entry.");
+    let e2 = server_b_txn
+        .internal_search_all_uuid(UUID_DOMAIN_INFO)
+        .expect("Unable to access entry.");
+
+    assert!(e1 == e2);
+
+    let e1_cs = e1.get_changestate();
+    let e2_cs = e2.get_changestate();
+    assert!(e1_cs == e2_cs);
+
+    // Check that an existing user was updated properly.
+    let e1 = server_a_txn
+        .internal_search_all_uuid(UUID_ADMIN)
+        .expect("Unable to access new entry.");
+    let e2 = server_b_txn
+        .internal_search_all_uuid(UUID_ADMIN)
+        .expect("Unable to access entry.");
+
+    let vx1 = e1.get_ava_single("spn").expect("spn not present");
+    let ex1 = Value::new_spn_str("admin", "new.example.com");
+    assert!(vx1 == ex1);
+
+    trace!(?e1);
+    trace!(?e2);
+    assert!(e1 == e2);
+
+    // Due to the domain rename, the spn regens on everything. This only occurs
+    // once per-replica, and is not unlimited.
+    let e1_cs = e1.get_changestate();
+    let e2_cs = e2.get_changestate();
+
+    trace!(?e1_cs);
+    trace!(?e2_cs);
+    assert!(e1_cs != e2_cs);
+
+    // Check that the user on server_b had it's spn regenerated too.
+    assert_eq!(
+        server_a_txn.internal_search_uuid(t_uuid),
+        Err(OperationError::NoMatchingEntries)
+    );
+
+    let e2 = server_b_txn
+        .internal_search_all_uuid(t_uuid)
+        .expect("Unable to access entry.");
+
+    let vx2 = e2.get_ava_single("spn").expect("spn not present");
+    let ex2 = Value::new_spn_str("testperson1", "new.example.com");
+    assert!(vx2 == ex2);
+
+    server_b_txn.commit().expect("Failed to commit");
+    drop(server_a_txn);
+
+    // Now we have to check a bunch of things are correct after the domain
+    // rename has completed. Generally this is that the spn is now correct and
+    // our other configs have reloaded.
+    //
+    // Possible to check the webauthn rp_id?
+
+    let mut server_a_txn = server_a.write(duration_from_epoch_now()).await;
+    let mut server_b_txn = server_b.read().await;
+
+    trace!("========================================");
+    //               from               to
+    repl_incremental(&mut server_b_txn, &mut server_a_txn);
+
+    // Check the admin is now in sync
+    let e1 = server_a_txn
+        .internal_search_all_uuid(UUID_ADMIN)
+        .expect("Unable to access new entry.");
+    let e2 = server_b_txn
+        .internal_search_all_uuid(UUID_ADMIN)
+        .expect("Unable to access entry.");
+
+    let vx1 = e1.get_ava_single("spn").expect("spn not present");
+    let ex1 = Value::new_spn_str("admin", "new.example.com");
+    assert!(vx1 == ex1);
+    assert!(e1 == e2);
+
+    let e1_cs = e1.get_changestate();
+    let e2_cs = e2.get_changestate();
+    assert!(e1_cs == e2_cs);
+
+    // Check the test person is back over and now in sync.
+    let e1 = server_a_txn
+        .internal_search_all_uuid(t_uuid)
+        .expect("Unable to access entry.");
+    let e2 = server_b_txn
+        .internal_search_all_uuid(t_uuid)
+        .expect("Unable to access entry.");
+
+    let vx2 = e2.get_ava_single("spn").expect("spn not present");
+    let ex2 = Value::new_spn_str("testperson1", "new.example.com");
+    assert!(vx2 == ex2);
+    assert!(e1 == e2);
+
+    let e1_cs = e1.get_changestate();
+    let e2_cs = e2.get_changestate();
+    assert!(e1_cs == e2_cs);
+
+    server_a_txn.commit().expect("Failed to commit");
+    drop(server_b_txn);
+}
 
 // Test schema addition / change over incremental.
 

server/lib/src/server/mod.rs

@@ -114,7 +114,7 @@ pub struct QueryServerWriteTransaction<'a> {
     pub(crate) changed_acp: bool,
     pub(crate) changed_oauth2: bool,
     pub(crate) changed_domain: bool,
-    changed_sync_agreement: bool,
+    pub(crate) changed_sync_agreement: bool,
     // Store the list of changed uuids for other invalidation needs?
     pub(crate) changed_uuid: HashSet<Uuid>,
     _db_ticket: SemaphorePermit<'a>,

server/lib/src/value.rs

@@ -25,7 +25,7 @@ use sshkeys::PublicKey as SshPublicKey;
 use time::OffsetDateTime;
 use url::Url;
 use uuid::Uuid;
-use webauthn_rs::prelude::{DeviceKey as DeviceKeyV4, Passkey as PasskeyV4};
+use webauthn_rs::prelude::{AttestedPasskey as DeviceKeyV4, Passkey as PasskeyV4};
 
 use crate::be::dbentry::DbIdentSpn;
 use crate::credential::{totp::Totp, Credential};

server/lib/src/valueset/cred.rs

@@ -1,7 +1,7 @@
 use std::collections::btree_map::Entry as BTreeEntry;
 use std::collections::BTreeMap;
 
-use webauthn_rs::prelude::{DeviceKey as DeviceKeyV4, Passkey as PasskeyV4};
+use webauthn_rs::prelude::{AttestedPasskey as DeviceKeyV4, Passkey as PasskeyV4};
 
 use crate::be::dbvalue::{
     DbValueCredV1, DbValueDeviceKeyV1, DbValueIntentTokenStateV1, DbValuePasskeyV1,

server/lib/src/valueset/mod.rs

@@ -9,7 +9,7 @@ use openssl::pkey::Public;
 use smolset::SmolSet;
 use time::OffsetDateTime;
 // use std::fmt::Debug;
-use webauthn_rs::prelude::DeviceKey as DeviceKeyV4;
+use webauthn_rs::prelude::AttestedPasskey as DeviceKeyV4;
 use webauthn_rs::prelude::Passkey as PasskeyV4;
 
 use kanidm_proto::v1::Filter as ProtoFilter;

server/testkit/tests/proto_v1_test.rs

@@ -1181,7 +1181,7 @@ async fn setup_demo_account_passkey(rsclient: &KanidmClient) -> WebauthnAuthenti
         .unwrap();
 
     // Setup and update the passkey
-    let mut wa = WebauthnAuthenticator::new(SoftPasskey::new());
+    let mut wa = WebauthnAuthenticator::new(SoftPasskey::new(true));
 
     let status = rsclient
         .idm_account_credential_update_passkey_init(&session_token)