Repair systemd reload notifications (#3355)

In order for the RELOAD and the subsequent READY notifications to be
correctly processed, the RELOAD notification must be accompanied with a
MONOTONIC_USEC one.

.github/workflows/clippy.yml

@@ -27,11 +27,13 @@ jobs:
           sudo apt-get update && \
           sudo apt-get install -y \
             libpam0g-dev \
-            libudev-dev \
+            libselinux1-dev \
             libssl-dev \
+            libsystemd-dev \
+            libtss2-dev \
+            libudev-dev \
             pkg-config \
-            tpm-udev \
+            tpm-udev
-            libtss2-dev
       - name: "Run clippy"
         run: cargo clippy --lib --bins --examples --all-features
   fmt:

.github/workflows/rust_build.yml

@@ -37,7 +37,8 @@ jobs:
           sudo apt-get install -y \
             libpam0g-dev \
             libudev-dev \
-            libssl-dev
+            libssl-dev \
+            libsystemd-dev
 
       - name: "Build the workspace"
         run: cargo build --workspace
@@ -84,7 +85,8 @@ jobs:
           sudo apt-get install -y \
             libpam0g-dev \
             libudev-dev \
-            libssl-dev
+            libssl-dev \
+            libsystemd-dev
 
       - name: "Build the workspace"
         run: cargo build --workspace
@@ -127,6 +129,7 @@ jobs:
             libpam0g-dev \
             libudev-dev \
             libssl-dev \
+            libsystemd-dev \
             ripgrep
       - name: "Run the release build test script"
         env:

Cargo.lock

@@ -5106,9 +5106,12 @@ dependencies = [
 
 [[package]]
 name = "sd-notify"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1be20c5f7f393ee700f8b2f28ea35812e4e212f40774b550cd2a93ea91684451"
+checksum = "561e6b346a5e59e0b8a07894004897d7160567e3352d2ebd6c3741d4e086b6f5"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "security-framework"

Cargo.toml

@@ -243,7 +243,7 @@ rustls = { version = "0.23.20", default-features = false, features = [
     "aws_lc_rs",
 ] }
 
-sd-notify = "^0.4.3"
+sd-notify = "^0.4.4"
 selinux = "^0.4.6"
 serde = "^1.0.217"
 serde_cbor = { version = "0.12.0-dev", package = "serde_cbor_2" }

book/src/developers/readme.md

@@ -103,7 +103,7 @@ You will need [rustup](https://rustup.rs/) to install a Rust toolchain.
 You will need to install rustup and our build dependencies with:
 
 ```bash
-zypper in rustup git libudev-devel sqlite3-devel libopenssl-3-devel libselinux-devel pam-devel tpm2-0-tss-devel
+zypper in rustup git libudev-devel sqlite3-devel libopenssl-3-devel libselinux-devel pam-devel systemd-devel tpm2-0-tss-devel
 ```
 
 You can then use rustup to complete the setup of the toolchain.
@@ -157,7 +157,7 @@ You need [rustup](https://rustup.rs/) to install a Rust toolchain.
 You will also need some system libraries to build this, which can be installed by running:
 
 ```bash
-sudo apt-get install libudev-dev libssl-dev pkg-config libpam0g-dev
+sudo apt-get install libudev-dev libssl-dev libsystemd-dev pkg-config libpam0g-dev
 ```
 
 Tested with Ubuntu 20.04 and 22.04.

scripts/devcontainer_postcreate.sh

@@ -18,6 +18,7 @@ sudo apt-get install -y \
     jq \
     libpam0g-dev \
     libssl-dev \
+    libsystemd-dev \
     libudev-dev \
     pkg-config \
     ripgrep

scripts/install_ubuntu_dependencies.sh

@@ -13,6 +13,7 @@ ${SUDOCMD} apt-get update &&
         libpam0g-dev \
         libudev-dev \
         libssl-dev \
+        libsystemd-dev \
         pkg-config \
         curl \
         rsync \

server/Dockerfile

@@ -29,6 +29,7 @@ RUN \
         libopenssl-3-devel \
         pam-devel \
         sqlite3-devel \
+        systemd-devel \
         rsync \
         findutils \
         which \

server/daemon/src/main.rs

@@ -751,7 +751,21 @@ async fn kanidm_main(
             if !config_test {
                 // On linux, notify systemd.
                 #[cfg(target_os = "linux")]
+                {
                     let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]);
+                    // Undocumented systemd feature - all messages should have a monotonic usec sent
+                    // with them. In some cases like "reloading" messages, it is undocumented but
+                    // failure to send this message causes the reload to fail.
+                    if let Ok(monotonic_usec) = sd_notify::NotifyState::monotonic_usec_now() {
+                        let _ = sd_notify::notify(true, &[monotonic_usec]);
+                    } else {
+                        error!("CRITICAL!!! Unable to access clock monotonic time. SYSTEMD WILL KILL US.");
+                    };
+                    let _ = sd_notify::notify(
+                        true,
+                        &[sd_notify::NotifyState::Status("Started Kanidm 🦀")],
+                    );
+                };
 
                 match sctx {
                     Ok(mut sctx) => {
@@ -785,7 +799,18 @@ async fn kanidm_main(
                                                     // Reload TLS certificates
                                                     // systemd has a special reload handler for this.
                                                     #[cfg(target_os = "linux")]
+                                                    {
                                                     let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Reloading]);
+                                                    // CRITICAL - if you do not send a monotonic usec message after a reloading
+                                                    // message, your service WILL BE KILLED.
+                                if let Ok(monotonic_usec) = sd_notify::NotifyState::monotonic_usec_now() {
+                                let _ =
+                                    sd_notify::notify(true, &[monotonic_usec]);
+                                } else {
+                                    error!("CRITICAL!!! Unable to access clock monotonic time. SYSTEMD WILL KILL US.");
+                                };
+                                                    let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Status("Reloading ...")]);
+                                                    }
 
                                                     sctx.tls_acceptor_reload().await;
 
@@ -794,7 +819,16 @@ async fn kanidm_main(
                                                     tokio::time::sleep(std::time::Duration::from_secs(5)).await;
 
                                                     #[cfg(target_os = "linux")]
+                                                    {
                                                     let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]);
+                                if let Ok(monotonic_usec) = sd_notify::NotifyState::monotonic_usec_now() {
+                                let _ =
+                                    sd_notify::notify(true, &[monotonic_usec]);
+                                } else {
+                                    error!("CRITICAL!!! Unable to access clock monotonic time. SYSTEMD WILL KILL US.");
+                                };
+                                                    let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Status("Reload Success")]);
+                                                    }
 
                                                     info!("Reload complete");
                                                 }