adding compression middleware to tide (#878)

* adding compression middleware to tide
* added notes, tests for regex things

Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
 [[package]]
 name = "aead"
 version = "0.3.2"
@@ -115,6 +121,16 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
+[[package]]
+name = "async-attributes"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "async-channel"
 version = "1.6.1"
@@ -126,6 +142,19 @@ dependencies = [
  "futures-core",
 ]
 
+[[package]]
+name = "async-compression"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"
+dependencies = [
+ "flate2",
+ "futures-core",
+ "futures-io",
+ "memchr",
+ "pin-project-lite 0.2.9",
+]
+
 [[package]]
 name = "async-dup"
 version = "1.2.2"
@@ -268,6 +297,7 @@ version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d"
 dependencies = [
+ "async-attributes",
  "async-channel",
  "async-global-executor",
  "async-io",
@@ -308,6 +338,19 @@ version = "4.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9"
 
+[[package]]
+name = "async-tls"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d85a97c4a0ecce878efd3f945f119c78a646d8975340bca0398f9bb05c30cc52"
+dependencies = [
+ "futures-core",
+ "futures-io",
+ "rustls",
+ "webpki",
+ "webpki-roots",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.56"
@@ -715,6 +758,17 @@ dependencies = [
  "cache-padded",
 ]
 
+[[package]]
+name = "config"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3"
+dependencies = [
+ "lazy_static",
+ "nom 5.1.2",
+ "serde",
+]
+
 [[package]]
 name = "console"
 version = "0.15.0"
@@ -827,6 +881,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba"
 
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "criterion"
 version = "0.3.5"
@@ -1055,6 +1118,30 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "dashmap"
+version = "4.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
+dependencies = [
+ "cfg-if 1.0.0",
+ "num_cpus",
+]
+
+[[package]]
+name = "deadpool"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d126179d86aee4556e54f5f3c6bf6d9884e7cc52cef82f77ee6f90a7747616d"
+dependencies = [
+ "async-trait",
+ "config",
+ "crossbeam-queue",
+ "num_cpus",
+ "serde",
+ "tokio",
+]
+
 [[package]]
 name = "derive_builder"
 version = "0.11.2"
@@ -1283,6 +1370,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "flate2"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1784,10 +1881,17 @@ version = "6.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e023af341b797ce2c039f7c6e1d347b68d0f7fd0bc7ac234fe69cfadcca1f89a"
 dependencies = [
+ "async-h1",
+ "async-std",
+ "async-tls",
  "async-trait",
  "cfg-if 1.0.0",
+ "dashmap",
+ "deadpool",
+ "futures",
  "http-types",
  "log",
+ "rustls",
 ]
 
 [[package]]
@@ -2328,12 +2432,31 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
+[[package]]
+name = "mime_guess"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
+dependencies = [
+ "mime",
+ "unicase",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.3"
@@ -2671,6 +2794,35 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
+[[package]]
+name = "phf_codegen"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
+dependencies = [
+ "phf_shared",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "phonenumber"
 version = "0.3.1+8.12.9"
@@ -3125,6 +3277,21 @@ dependencies = [
  "winreg",
 ]
 
+[[package]]
+name = "ring"
+version = "0.16.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin",
+ "untrusted",
+ "web-sys",
+ "winapi",
+]
+
 [[package]]
 name = "route-recognizer"
 version = "0.2.0"
@@ -3188,6 +3355,19 @@ dependencies = [
  "semver 1.0.9",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81"
+dependencies = [
+ "base64 0.12.3",
+ "log",
+ "ring",
+ "sct",
+ "webpki",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.10"
@@ -3262,10 +3442,12 @@ dependencies = [
  "oauth2",
  "openssl",
  "profiles",
+ "regex",
  "reqwest",
  "serde",
  "serde_json",
  "tide",
+ "tide-compress",
  "tide-openssl",
  "tokio",
  "tokio-openssl",
@@ -3277,6 +3459,16 @@ dependencies = [
  "webauthn-authenticator-rs",
 ]
 
+[[package]]
+name = "sct"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "security-framework"
 version = "2.6.1"
@@ -3512,6 +3704,12 @@ dependencies = [
  "event-listener",
 ]
 
+[[package]]
+name = "siphasher"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
+
 [[package]]
 name = "slab"
 version = "0.4.6"
@@ -3558,6 +3756,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "sshkeys"
 version = "0.3.2"
@@ -3645,6 +3849,28 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
+[[package]]
+name = "surf"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "718b1ae6b50351982dedff021db0def601677f2120938b070eadb10ba4038dd7"
+dependencies = [
+ "async-std",
+ "async-trait",
+ "cfg-if 1.0.0",
+ "futures-util",
+ "getrandom 0.2.6",
+ "http-client",
+ "http-types",
+ "log",
+ "mime_guess",
+ "once_cell",
+ "pin-project-lite 0.2.9",
+ "rustls",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "sval"
 version = "1.0.0-alpha.5"
@@ -3777,6 +4003,24 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "tide-compress"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97522b129f68a4b1c5399638fc04893350eab3299e71912325705ba937206ced"
+dependencies = [
+ "async-compression",
+ "async-std",
+ "futures-lite",
+ "http-types",
+ "phf_codegen",
+ "regex",
+ "serde",
+ "serde_json",
+ "surf",
+ "tide",
+]
+
 [[package]]
 name = "tide-openssl"
 version = "0.1.1"
@@ -4081,6 +4325,15 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
 
+[[package]]
+name = "unicase"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.8"
@@ -4124,6 +4377,12 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "url"
 version = "2.2.2"
@@ -4399,6 +4658,25 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "webpki"
+version = "0.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f"
+dependencies = [
+ "webpki",
+]
+
 [[package]]
 name = "wepoll-ffi"
 version = "0.1.2"

kanidmd/score/Cargo.toml

@@ -23,10 +23,13 @@ kanidm_proto = { path = "../../kanidm_proto" }
 ldap3_proto = "^0.2.3"
 libc = "^0.2.126"
 openssl = "^0.10.38"
+regex = "1.5.6"
 serde = { version = "^1.0.137", features = ["derive"] }
 serde_json = "^1.0.81"
 tide = "^0.16.0"
 tide-openssl = "^0.1.1"
+# I tried including brotli and it didn't work, including "default" pulls a mime-type list from the internet on build
+tide-compress = { version = "0.10.3", default-features = false, features = [ "deflate", "gzip", "regex-check" ] }
 tokio = { version = "^1.19.1", features = ["net", "sync", "io-util", "macros"] }
 tokio-openssl = "^0.6.3"
 tokio-util = { version = "^0.7.3", features = ["codec"] }

kanidmd/score/src/https/mod.rs

@@ -10,15 +10,16 @@ use kanidm::config::{ServerRole, TlsConfiguration};
 use kanidm::prelude::*;
 use kanidm::status::StatusActor;
 
+use regex::Regex;
 use serde::Serialize;
 use std::fs::canonicalize;
 use std::path::PathBuf;
 use std::str::FromStr;
 use uuid::Uuid;
 
-use tide_openssl::TlsListener;
-
 use kanidm::tracing_tree::TreeMiddleware;
+use tide_compress::CompressMiddleware;
+use tide_openssl::TlsListener;
 use tracing::{error, info};
 
 use compact_jwt::{Jws, JwsSigner, JwsUnverified, JwsValidator};
@@ -33,6 +34,34 @@ pub struct AppState {
     pub jws_validator: std::sync::Arc<JwsValidator>,
 }
 
+/// This is for the tide_compression middleware so that we only compress certain content types.
+///
+/// ```
+/// use score::https::compression_content_type_checker;
+/// let these_should_match = vec![
+///     "application/wasm",
+///     "text/json",
+///     "text/javascript"
+/// ];
+/// for test_value in these_should_match {
+///     eprintln!("checking {:?}", test_value);
+///     assert!(compression_content_type_checker().is_match(test_value));
+/// }
+/// assert!(compression_content_type_checker().is_match("application/wasm"));
+/// let these_should_fail = vec![
+///     "image/jpeg",
+///     "image/wasm",
+///     "text/html",
+/// ];
+/// for test_value in these_should_fail {
+///     eprintln!("checking {:?}", test_value);
+///     assert!(!compression_content_type_checker().is_match(test_value));
+/// }
+/// ```
+pub fn compression_content_type_checker() -> Regex {
+    Regex::new(r"^(?:(image/svg\+xml)|(?:application|text)/(?:css|javascript|json|text|xml|wasm))$")
+        .expect("regex matcher for tide_compress content-type check failed to compile")
+}
 pub trait RequestExtensions {
     fn get_current_uat(&self) -> Option<String>;
 
@@ -433,12 +462,38 @@ pub fn create_https_server(
         }
         info!("Web UI package path: {:?}", canonicalize(pkg_path).unwrap());
 
+        /*
+        Let's build a compression middleware!
+
+        The threat of the TLS BREACH attack [1] was considered as part of adding
+        the CompressMiddleware configuration.
+
+        The attack targets secrets compressed and encrypted in flight with the intent
+        to infer their content.
+
+        This is not a concern for the paths covered by this configuration
+        ( /, /ui/<and all sub-paths>, /pkg/<and all sub-paths> ),
+        as they're all static content with no secrets in transit - all that data should
+        come from Kanidm's REST API, which is on a different path and not covered by
+        the compression middleware.
+
+
+        [1] - https://resources.infosecinstitute.com/topic/the-breach-attack/
+        */
+
+        let compress_middleware = CompressMiddleware::builder()
+            .threshold(1024)
+            .content_type_check(Some(compression_content_type_checker()))
+            .build();
+
         let mut static_tserver = tserver.at("");
         static_tserver.with(StaticContentMiddleware::default());
         static_tserver.with(UIContentSecurityPolicyResponseMiddleware::new(
             generate_integrity_hash(env!("KANIDM_WEB_UI_PKG_PATH").to_owned() + "/wasmloader.js")
                 .unwrap(),
         ));
+        // The compression middleware needs to be the last one added before routes
+        static_tserver.with(compress_middleware.clone());
 
         static_tserver.at("/").get(index_view);
         static_tserver.at("/ui/").get(index_view);
@@ -446,6 +501,8 @@ pub fn create_https_server(
 
         let mut static_dir_tserver = tserver.at("");
         static_dir_tserver.with(StaticContentMiddleware::default());
+        // The compression middleware needs to be the last one added before routes
+        static_dir_tserver.with(compress_middleware);
         static_dir_tserver
             .at("/pkg")
             .serve_dir(env!("KANIDM_WEB_UI_PKG_PATH"))

kanidmd/score/src/lib.rs

@@ -25,7 +25,7 @@ extern crate tracing;
 #[macro_use]
 extern crate kanidm;
 
-mod https;
+pub mod https;
 mod ldaps;
 
 // use crossbeam::channel::unbounded;