Improve handling of openssl3 in md4 tests (#1171)

Cargo.lock

@@ -2395,6 +2395,7 @@ dependencies = [
  "libsqlite3-sys",
  "num_enum",
  "openssl",
+ "openssl-sys",
  "profiles",
  "r2d2",
  "r2d2_sqlite",

Cargo.toml

@@ -91,6 +91,7 @@ lru = "^0.8.0"
 mathru = "^0.13.0"
 num_enum = "^0.5.7"
 oauth2_ext = { version = "^4.1.0", package = "oauth2" }
+openssl-sys = "^0.9"
 openssl = "^0.10.41"
 paste = "^1.0.9"
 pkg-config = "^0.3.26"

kanidmd/lib/Cargo.toml

@@ -41,6 +41,9 @@ ldap3_proto.workspace = true
 libc.workspace = true
 libsqlite3-sys.workspace = true
 num_enum.workspace = true
+# We need to explicitly ask for openssl-sys so that we get the version propogated
+# into the build.rs for legacy feature checks.
+openssl-sys.workspace = true
 openssl.workspace = true
 r2d2.workspace = true
 r2d2_sqlite.workspace = true

kanidmd/lib/build.rs

@@ -1,5 +1,15 @@
 // include!("src/lib/audit_loglevel.rs");
 
+use std::env;
+
 fn main() {
+    if let Ok(v) = env::var("DEP_OPENSSL_VERSION_NUMBER") {
+        let version = u64::from_str_radix(&v, 16).unwrap();
+
+        if version >= 0x3_00_00_00_0 {
+            println!("cargo:rustc-cfg=openssl3");
+        }
+    }
+
     profiles::apply_profile();
 }

kanidmd/lib/src/credential/mod.rs

@@ -387,12 +387,18 @@ impl Password {
                     .collect();
 
                 let dgst = MessageDigest::from_nid(Nid::MD4).ok_or_else(|| {
-                    error!("Unable to access MD4 - fips mode enabled?");
+                    error!("Unable to access MD4 - fips mode may be enabled, or you may need to activate the legacy provider.");
+                    error!("For more details, see https://wiki.openssl.org/index.php/OpenSSL_3.0#Providers");
                     OperationError::CryptographyError
                 })?;
 
                 hash::hash(dgst, &clear_utf16le)
-                    .map_err(|_| OperationError::CryptographyError)
+                    .map_err(|e| {
+                        debug!(?e);
+                        error!("Unable to digest MD4 - fips mode may be enabled, or you may need to activate the legacy provider.");
+                        error!("For more details, see https://wiki.openssl.org/index.php/OpenSSL_3.0#Providers");
+                        OperationError::CryptographyError
+                    })
                     .map(|chal_key| chal_key.as_ref() == key)
             }
         }
@@ -1196,23 +1202,65 @@ mod tests {
     }
     */
 
+    /*
+     * wbrown - 20221104 - I tried to programatically enable the legacy provider, but
+     * it consistently "did nothing at all", meaning we have to rely on users to enable
+     * this for this test.
+     */
+
+    /*
+    #[cfg(openssl3)]
+    fn setup_openssl_legacy_provider() -> openssl::lib_ctx::LibCtx {
+        let ctx = openssl::lib_ctx::LibCtx::new()
+            .expect("Failed to create new library context");
+
+        openssl::provider::Provider::load(Some(&ctx), "legacy")
+            .expect("Failed to setup provider.");
+
+        eprintln!("setup legacy provider maybe??");
+
+        ctx
+    }
+    */
+
     #[test]
     fn test_password_from_ipa_nt_hash() {
+        let _ = sketching::test_init();
         // Base64 no pad
         let im_pw = "ipaNTHash: iEb36u6PsRetBr3YMLdYbA";
         let password = "password";
         let r = Password::try_from(im_pw).expect("Failed to parse");
         assert!(r.requires_upgrade());
-        assert!(r.verify(password).unwrap_or(false));
+
+        match r.verify(password) {
+            Ok(r) => assert!(r),
+            Err(_) => {
+                if cfg!(openssl3) {
+                    warn!("To run this test, enable the legacy provider.");
+                } else {
+                    assert!(false);
+                }
+            }
+        }
     }
 
     #[test]
     fn test_password_from_samba_nt_hash() {
+        let _ = sketching::test_init();
         // Base64 no pad
         let im_pw = "sambaNTPassword: 8846F7EAEE8FB117AD06BDD830B7586C";
         let password = "password";
         let r = Password::try_from(im_pw).expect("Failed to parse");
         assert!(r.requires_upgrade());
-        assert!(r.verify(password).unwrap_or(false));
+        match r.verify(password) {
+            Ok(r) => assert!(r),
+            Err(_) => {
+                if cfg!(openssl3) {
+                    warn!("To run this test, enable the legacy provider.");
+                } else {
+                    assert!(false);
+                }
+            }
+        }
     }
 }