1355 docker builds (#1384)

.dockerignore

@@ -1,10 +1,12 @@
 .git
 .github
 .gitignore
+.mypy_cache
 .vscode
 *.md
+**/__pycache__/**
+docs
+kanidmd/sampledata
 Makefile
 target
-docs
 test.db
-kanidmd/sampledata

Makefile

@@ -5,6 +5,7 @@ IMAGE_ARCH ?= "linux/amd64,linux/arm64"
 CONTAINER_BUILD_ARGS ?=
 MARKDOWN_FORMAT_ARGS ?= --options-line-width=100
 CONTAINER_TOOL ?= docker
+BUILDKIT_PROGRESS ?= plain
 
 BOOK_VERSION ?= master
 
@@ -15,37 +16,37 @@ help:
 
 .PHONY: buildx/kanidmd/x86_64_v3
 buildx/kanidmd/x86_64_v3: ## build multiarch server images
-buildx/kanidmd/x86_64_v3: vendor
+buildx/kanidmd/x86_64_v3:
 	@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) --pull --push --platform "linux/amd64/v3" \
 		-f kanidmd/Dockerfile -t $(IMAGE_BASE)/server:x86_64_$(IMAGE_VERSION) \
+		--progress $(BUILDKIT_PROGRESS) \
 		--build-arg "KANIDM_BUILD_PROFILE=container_x86_64_v3" \
 		--build-arg "KANIDM_FEATURES=" \
 		$(CONTAINER_BUILD_ARGS) .
-	@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/server:$(IMAGE_VERSION)
 
 .PHONY: buildx/kanidmd
 buildx/kanidmd: ## Build multiarch kanidm server images and push to docker hub
-buildx/kanidmd: vendor
+buildx/kanidmd:
 	@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) \
 		--pull --push --platform $(IMAGE_ARCH) \
 		-f kanidmd/Dockerfile \
 		-t $(IMAGE_BASE)/server:$(IMAGE_VERSION) \
+		--progress $(BUILDKIT_PROGRESS) \
 		--build-arg "KANIDM_BUILD_PROFILE=container_generic" \
 		--build-arg "KANIDM_FEATURES=" \
 		$(CONTAINER_BUILD_ARGS) .
-	@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/server:$(IMAGE_VERSION)
 
 .PHONY: buildx/kanidm_tools
 buildx/kanidm_tools: ## Build multiarch kanidm tool images and push to docker hub
-buildx/kanidm_tools: vendor
+buildx/kanidm_tools:
 	@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) \
 		--pull --push --platform $(IMAGE_ARCH) \
 		-f kanidm_tools/Dockerfile \
 		-t $(IMAGE_BASE)/tools:$(IMAGE_VERSION) \
+		--progress $(BUILDKIT_PROGRESS) \
 		--build-arg "KANIDM_BUILD_PROFILE=container_generic" \
 		--build-arg "KANIDM_FEATURES=" \
 		$(CONTAINER_BUILD_ARGS) .
-	@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/tools:$(IMAGE_VERSION)
 
 .PHONY: buildx/radiusd
 buildx/radiusd: ## Build multi-arch radius docker images and push to docker hub
@@ -53,8 +54,8 @@ buildx/radiusd:
 	@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) \
 		--pull --push --platform $(IMAGE_ARCH) \
 		-f kanidm_rlm_python/Dockerfile \
+		--progress $(BUILDKIT_PROGRESS) \
 		-t $(IMAGE_BASE)/radius:$(IMAGE_VERSION) .
-	@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/radius:$(IMAGE_VERSION)
 
 .PHONY: buildx
 buildx: buildx/kanidmd/x86_64_v3 buildx/kanidmd buildx/kanidm_tools buildx/radiusd
@@ -114,11 +115,6 @@ install-tools: ## install kanidm_tools in your local environment
 install-tools:
 	cd kanidm_tools && cargo install --path . --force
 
-.PHONY: prep
-prep:
-	cargo outdated -R
-	cargo audit
-
 .PHONY: codespell
 codespell:
 	codespell -c \
@@ -215,6 +211,11 @@ docs/pykanidm/serve:
 
 ########################################################################
 
+.PHONY: release/prep
+prep:
+	cargo outdated -R
+	cargo audit
+
 .PHONY: release/kanidm
 release/kanidm: ## Build the Kanidm CLI - ensure you include the environment variable KANIDM_BUILD_PROFILE
 	cargo build -p kanidm_tools --bin kanidm --release
@@ -241,7 +242,6 @@ release/kanidm-unixd:
 		--bin kanidm_cache_clear \
 		--bin kanidm_cache_invalidate
 
-
 # cert things
 
 .PHONY: cert/clean

RELEASE_NOTES.md

@@ -16,13 +16,13 @@ report it to our [issue tracker].
 
 ## 2023-02-01 - Kanidm 1.1.0-alpha11
 
-This is the eleventh alpha series release of the Kanidm Identity Management project. Alpha releases are
+This is the eleventh alpha series release of the Kanidm Identity Management project. Alpha releases
-to help get feedback and ideas from the community on how we can continue to make this project better
+are to help get feedback and ideas from the community on how we can continue to make this project
-for a future supported release.
+better for a future supported release.
 
-The project is shaping up very nicely, and a beta will be coming soon! The main reason we haven't done
+The project is shaping up very nicely, and a beta will be coming soon! The main reason we haven't
-so yet is we haven't decided if we want to commit to the current API layout and freeze it yet. There
+done so yet is we haven't decided if we want to commit to the current API layout and freeze it yet.
-are still things we want to change there. Otherwise the server is stable and reliable.
+There are still things we want to change there. Otherwise the server is stable and reliable.
 
 ### Release Highlights
 

kanidm_rlm_python/Dockerfile

@@ -1,10 +1,20 @@
-FROM opensuse/tumbleweed:latest
+ARG BASE_IMAGE=opensuse/tumbleweed:latest
+FROM ${BASE_IMAGE} AS repos
+RUN \
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
+    zypper mr -k repo-oss && \
+    zypper mr -k repo-update && \
+    zypper dup -y
+
+# ======================
+FROM repos
 
 EXPOSE 1812 1813
 
 # These all need to be on one line else the rpm cache ends
 # up in the layers.
-RUN zypper refresh --force && \
+RUN \
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
     zypper install -y \
     freeradius-client \
     freeradius-server \
@@ -18,8 +28,7 @@ RUN zypper refresh --force && \
     iproute2 \
     iputils \
     openssl \
-    curl && \
+    curl
-    zypper clean
 
 ADD kanidm_rlm_python/mods-available/ /etc/raddb/mods-available/
 COPY kanidm_rlm_python/sites-available/ /etc/raddb/sites-available/
@@ -28,29 +37,30 @@ COPY kanidm_rlm_python/sites-available/ /etc/raddb/sites-available/
 WORKDIR /etc/raddb
 
 # Enable the python and cache module.
-RUN ln -s /etc/raddb/mods-available/python3 /etc/raddb/mods-enabled/python3
+RUN ln -s /etc/raddb/mods-available/python3 /etc/raddb/mods-enabled/python3 && \
-RUN ln -s /etc/raddb/sites-available/check-eap-tls /etc/raddb/sites-enabled/check-eap-tls
+    ln -s /etc/raddb/sites-available/check-eap-tls /etc/raddb/sites-enabled/check-eap-tls
 
 # disable auth via methods we don't support!
-RUN rm /etc/raddb/mods-available/sql
+RUN rm /etc/raddb/mods-available/sql && \
-RUN rm /etc/raddb/mods-enabled/{passwd,totp}
+    rm /etc/raddb/mods-enabled/{passwd,totp}
 
 # Allows the radiusd user to write to the directory
-RUN chown -R radiusd: /etc/raddb
+RUN chown -R radiusd: /etc/raddb && \
-RUN chmod 775 /etc/raddb/certs
+    chmod 775 /etc/raddb/certs && \
-RUN chmod 640 /etc/raddb/clients.conf
+    chmod 640 /etc/raddb/clients.conf
 
 RUN mkdir -p /pkg/pykanidm/
 COPY pykanidm/ /pkg/pykanidm/
 
 # install the package and its dependencies
-RUN python3 -m pip install --no-cache-dir --no-warn-script-location /pkg/pykanidm
+RUN python3 -m pip install --no-cache-dir --no-warn-script-location /pkg/pykanidm && \
-# clean up after install
+    rm -rf /pkg/*
-RUN rm -rf /pkg/*
+
+COPY kanidm_rlm_python/radius_entrypoint.py /radius_entrypoint.py
 
-USER radiusd
 ENV LD_PRELOAD=/usr/lib64/libpython3.so
 ENV KANIDM_CONFIG_FILE="/data/kanidm"
 
-COPY kanidm_rlm_python/radius_entrypoint.py /radius_entrypoint.py
+USER radiusd
+
 CMD [ "/usr/bin/python3", "/radius_entrypoint.py" ]

kanidm_tools/Dockerfile

@@ -1,19 +1,27 @@
 # This builds the kanidm CLI tools
 ARG BASE_IMAGE=opensuse/tumbleweed:latest
 FROM ${BASE_IMAGE} AS repos
-RUN zypper refresh --force
+RUN \
-RUN zypper dup -y
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
+    zypper mr -k repo-oss && \
+    zypper mr -k repo-non-oss && \
+    zypper mr -k repo-update && \
+    zypper dup -y
 
 FROM repos AS builder
 ARG KANIDM_FEATURES
 ARG KANIDM_BUILD_PROFILE
 ARG KANIDM_BUILD_OPTIONS=""
 
-RUN echo Profile $KANIDM_BUILD_PROFILE
+# Set the build profile
-RUN echo Features $KANIDM_FEATURES
+ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
+ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
 
-RUN zypper install -y --no-recommends \
+RUN \
-        rustup wasm-pack \
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
+    zypper install -y --no-recommends \
+        sccache \
+        cargo \
         clang \
         make automake autoconf \
         libopenssl-3-devel \
@@ -23,29 +31,18 @@ RUN zypper install -y --no-recommends \
         rsync \
         mold
 
-RUN zypper clean -a
-RUN rustup default stable
-
 COPY . /usr/src/kanidm
 
-RUN mkdir -p /usr/src/kanidm/.cargo
-RUN cp /usr/src/kanidm/cargo_vendor_config /usr/src/kanidm/.cargo/config.toml
-
 WORKDIR /usr/src/kanidm/
 
-# Set the build profile
-ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
-ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
-
 # build the CLI
-RUN if [ -z "${KANIDM_FEATURES}" ]; then \
+RUN \
-  cargo build -p kanidm_tools ${KANIDM_BUILD_OPTIONS} \
+--mount=type=cache,id=cargo,target=/cargo \
-    --target-dir="/usr/src/kanidm/target/" \
+--mount=type=cache,id=sccache,target=/sccache \
-    --release; \
+export CARGO_HOME=/cargo; \
-  cargo build -p kanidm-ipa-sync ${KANIDM_BUILD_OPTIONS} \
+export SCCACHE_DIR=/sccache; \
-    --target-dir="/usr/src/kanidm/target/" \
+export RUSTC_WRAPPER=/usr/bin/sccache; \
-    --release; \
+export CC="/usr/bin/clang"; \
-else \
     cargo build -p kanidm_tools ${KANIDM_BUILD_OPTIONS} \
         --target-dir="/usr/src/kanidm/target/" \
         --features="${KANIDM_FEATURES}" \
@@ -54,24 +51,24 @@ else \
         --target-dir="/usr/src/kanidm/target/" \
         --features="${KANIDM_FEATURES}" \
         --release; \
-fi
+    sccache -s
-
-RUN ls -al /usr/src/kanidm/target/release
 
 # == Construct the tools container
 FROM repos
 
-RUN zypper install -y timezone busybox-adduser openssl-3 && \
+ENV RUST_BACKTRACE 1
-    zypper clean -a
+
+RUN \
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
+    zypper install -y timezone busybox-adduser openssl-3
 
 COPY --from=builder /usr/src/kanidm/target/release/kanidm /sbin/
 COPY --from=builder /usr/src/kanidm/target/release/kanidm-ipa-sync /sbin/
 RUN chmod +x /sbin/kanidm
 RUN chmod +x /sbin/kanidm-ipa-sync
-ENV RUST_BACKTRACE 1
 
-RUN adduser -D -H kanidm
+RUN adduser -D -H kanidm && \
-RUN mkdir /etc/kanidm && \
+    mkdir /etc/kanidm && \
     touch /etc/kanidm/config
 
 USER kanidm

kanidmd/Dockerfile

@@ -1,8 +1,12 @@
 # Build the main Kanidmd server
 ARG BASE_IMAGE=opensuse/tumbleweed:latest
 FROM ${BASE_IMAGE} AS repos
-RUN zypper refresh --force
+RUN \
-RUN zypper dup -y
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
+    zypper mr -k repo-oss && \
+    zypper mr -k repo-non-oss && \
+    zypper mr -k repo-update && \
+    zypper dup -y
 
 # ======================
 FROM repos AS builder
@@ -10,11 +14,15 @@ ARG KANIDM_FEATURES
 ARG KANIDM_BUILD_PROFILE="container_generic"
 ARG KANIDM_BUILD_OPTIONS=""
 
-RUN echo Profile $KANIDM_BUILD_PROFILE
+# Set the build profile
-RUN echo Features $KANIDM_FEATURES
+ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
+ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
 
-RUN zypper install -y --no-recommends \
+RUN \
-        rustup \
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
+    zypper install -y --no-recommends \
+        sccache \
+        cargo \
         clang \
         make automake autoconf \
         libopenssl-3-devel pam-devel \
@@ -23,17 +31,9 @@ RUN zypper install -y --no-recommends \
         findutils \
         which \
         mold
-# wasm-pack \
-# lld
-
-RUN zypper clean -a
-RUN rustup default stable
 
 COPY . /usr/src/kanidm
 
-RUN mkdir -p /usr/src/kanidm/.cargo
-RUN cp /usr/src/kanidm/cargo_vendor_config /usr/src/kanidm/.cargo/config.toml
-
 # ======================
 
 # WORKDIR /usr/src/kanidm/kanidmd_web_ui
@@ -45,37 +45,31 @@ RUN cp /usr/src/kanidm/cargo_vendor_config /usr/src/kanidm/.cargo/config.toml
 
 WORKDIR /usr/src/kanidm/kanidmd/daemon
 
-# Set the build profile
-ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
-ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
-
 # Exports don't persist through RUN statements.
-RUN export CC="/usr/bin/clang"; \
+RUN \
-if [ -z "${KANIDM_FEATURES}" ]; then \
+--mount=type=cache,id=cargo,target=/cargo \
+--mount=type=cache,id=sccache,target=/sccache \
+export CARGO_HOME=/cargo; \
+export SCCACHE_DIR=/sccache; \
+export RUSTC_WRAPPER=/usr/bin/sccache; \
+export CC="/usr/bin/clang"; \
     cargo build -p daemon ${KANIDM_BUILD_OPTIONS} \
-    --locked --offline \
-    --target-dir="/usr/src/kanidm/target/" \
-    --release; \
-else \
-  cargo build -p daemon ${KANIDM_BUILD_OPTIONS} \
-    --locked --offline \
         --target-dir="/usr/src/kanidm/target/" \
         --features="${KANIDM_FEATURES}" \
         --release; \
-fi
+    sccache -s
-
-RUN ls -al /usr/src/kanidm/target/release
 
 # ======================
 
 FROM repos
 
-RUN zypper install -y \
+RUN \
+    --mount=type=cache,id=zypp,target=/var/cache/zypp \
+    zypper install -y \
         timezone \
         openssl-3 \
         sqlite3 \
         pam
-RUN zypper clean -a
 
 COPY --from=builder /usr/src/kanidm/target/release/kanidmd /sbin/
 COPY --from=builder /usr/src/kanidm/kanidmd_web_ui/pkg /pkg

kanidmd/lib/src/be/mod.rs

@@ -1714,9 +1714,9 @@ impl Backend {
         idxkeys: Vec<IdxKey>,
         vacuum: bool,
     ) -> Result<Self, OperationError> {
-        debug!("DB tickets -> {:?}", cfg.pool_size);
+        info!("DB tickets -> {:?}", cfg.pool_size);
-        debug!("Profile -> {}", env!("KANIDM_PROFILE_NAME"));
+        info!("Profile -> {}", env!("KANIDM_PROFILE_NAME"));
-        debug!("CPU Flags -> {}", env!("KANIDM_CPU_FLAGS"));
+        info!("CPU Flags -> {}", env!("KANIDM_CPU_FLAGS"));
 
         // If in memory, reduce pool to 1
         if cfg.path.is_empty() {

profiles/container_generic.toml

@@ -1,3 +1,3 @@
 web_ui_pkg_path = "/pkg"
-# Valid options are none, native, x86_64_v1, x86_64_v3
+# Don't set the cpu_flags to autodetect for this platform
-cpu_flags = "none"
+# cpu_flags = "none"

profiles/container_x86_64_v3.toml

@@ -1,3 +1,3 @@
 web_ui_pkg_path = "/pkg"
-# Valid options are none, native, x86_64, x86_64_v3
+# Define this to override the arch.
 cpu_flags = "x86_64_v3"

profiles/developer.toml

@@ -1,3 +1,3 @@
 web_ui_pkg_path = "../../kanidmd_web_ui/pkg"
-# Valid options are none, native, x86_64, x86_64_v3
+# Set to native for developer machines.
 cpu_flags = "native"

profiles/release_suse_generic.toml

@@ -1,2 +1,3 @@
 web_ui_pkg_path = "/usr/share/kanidm/ui/pkg"
-cpu_flags = "none"
+# Don't set the value for autodetect
+# cpu_flags = "none"

profiles/release_suse_x86_64.toml

@@ -1,2 +0,0 @@
-web_ui_pkg_path = "/usr/share/kanidm/ui/pkg"
-cpu_flags = "x86_64_v1"

profiles/src/lib.rs

@@ -7,16 +7,30 @@ use serde::Deserialize;
 enum CpuOptLevel {
     none,
     native,
-    x86_64_v1,
+    neon_v8,
+    x86_64_v2,
     x86_64_v3,
 }
 
+impl Default for CpuOptLevel {
+    fn default() -> Self {
+        if cfg!(target_arch = "x86_64") {
+            CpuOptLevel::x86_64_v2
+        } else if cfg!(target_arch = "aarch64") {
+            CpuOptLevel::neon_v8
+        } else {
+            CpuOptLevel::none
+        }
+    }
+}
+
 impl std::fmt::Display for CpuOptLevel {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match &self {
             CpuOptLevel::none => write!(f, "none"),
             CpuOptLevel::native => write!(f, "native"),
-            CpuOptLevel::x86_64_v1 => write!(f, "x86_64_v1"),
+            CpuOptLevel::neon_v8 => write!(f, "neon_v8"),
+            CpuOptLevel::x86_64_v2 => write!(f, "x86_64_v2"),
             CpuOptLevel::x86_64_v3 => write!(f, "x86_64_v3"),
         }
     }
@@ -25,6 +39,7 @@ impl std::fmt::Display for CpuOptLevel {
 #[derive(Debug, Deserialize)]
 struct ProfileConfig {
     web_ui_pkg_path: String,
+    #[serde(default)]
     cpu_flags: CpuOptLevel,
 }
 
@@ -43,17 +58,12 @@ pub fn apply_profile() {
     let profile_cfg: ProfileConfig = toml::from_slice(&data)
         .unwrap_or_else(|_| panic!("Failed to parse profile - {} - {}", profile, contents));
 
-    /*
-     *  x86-64: CMOV, CMPXCHG8B, FPU, FXSR, MMX, FXSR, SCE, SSE, SSE2
-     *  x86-64-v2: (close to Nehalem) CMPXCHG16B, LAHF-SAHF, POPCNT, SSE3, SSE4.1, SSE4.2, SSSE3
-     *  x86-64-v3: (close to Haswell) AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE, XSAVE
-     */
-
     match profile_cfg.cpu_flags {
         CpuOptLevel::none => {}
         CpuOptLevel::native => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-cpu=native"),
-        CpuOptLevel::x86_64_v1 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-feature=+cmov,+cx8,+fxsr,+mmx,+sse,+sse2"),
+        CpuOptLevel::neon_v8 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-features=+neon,+fp-armv8"),
-        CpuOptLevel::x86_64_v3 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-feature=+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+cx16,+sahf,+popcnt,+sse3,+sse4.1,+sse4.2,+avx,+avx2,+bmi,+bmi2,+f16c,+fma,+lzcnt,+movbe,+xsave"),
+        CpuOptLevel::x86_64_v2 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-cpu=x86-64-v2"),
+        CpuOptLevel::x86_64_v3 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-cpu=x86-64-v3"),
     }
     println!("cargo:rustc-env=KANIDM_PROFILE_NAME={}", profile);
     println!("cargo:rustc-env=KANIDM_CPU_FLAGS={}", profile_cfg.cpu_flags);

project_docs/RELEASE_CHECKLIST.md

@@ -47,8 +47,8 @@ cargo install cargo-outdated
 - [ ] git push origin 1.1.0-alpha.x
 - [ ] git push origin 1.1.0-alpha.x --tags
 
-- [ ] github -> create new release based on tag (not branch)
+- [ ] github -> create new release based on tag (not branch) - use tag because then tools will get
-      - use tag because then tools will get the tag + patches we apply.
+      the tag + patches we apply.
 
 ### Cargo publish