mart-w/kanidm
1
0
Fork 0
mirror of https://github.com/kanidm/kanidm.git synced 2025-02-22 20:26:30 +01:00
Code Issues Actions 16 Packages Projects Releases Wiki Activity

1355 docker builds (#1384)

Browse source
This commit is contained in:
Firstyear 2023-02-17 17:02:01 +10:00 committed by GitHub
parent 43ad0dc3d0
commit 0d8d9e1a62
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 146 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.dockerignore
View file

@ -1,10 +1,12 @@
.git
.github
.gitignore
.mypy_cache
.vscode
*.md
**/__pycache__/**
docs
kanidmd/sampledata
Makefile
target
docs
test.db
kanidmd/sampledata

26
Makefile
View file

@ -5,6 +5,7 @@ IMAGE_ARCH ?= "linux/amd64,linux/arm64"
CONTAINER_BUILD_ARGS ?=
MARKDOWN_FORMAT_ARGS ?= --options-line-width=100
CONTAINER_TOOL ?= docker
BUILDKIT_PROGRESS ?= plain
BOOK_VERSION ?= master
@ -15,37 +16,37 @@ help:
.PHONY: buildx/kanidmd/x86_64_v3
buildx/kanidmd/x86_64_v3: ## build multiarch server images
buildx/kanidmd/x86_64_v3: vendor
buildx/kanidmd/x86_64_v3:
@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) --pull --push --platform "linux/amd64/v3" \
-f kanidmd/Dockerfile -t $(IMAGE_BASE)/server:x86_64_$(IMAGE_VERSION) \
--progress $(BUILDKIT_PROGRESS) \
--build-arg "KANIDM_BUILD_PROFILE=container_x86_64_v3" \
--build-arg "KANIDM_FEATURES=" \
$(CONTAINER_BUILD_ARGS) .
@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/server:$(IMAGE_VERSION)
.PHONY: buildx/kanidmd
buildx/kanidmd: ## Build multiarch kanidm server images and push to docker hub
buildx/kanidmd: vendor
buildx/kanidmd:
@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) \
--pull --push --platform $(IMAGE_ARCH) \
-f kanidmd/Dockerfile \
-t $(IMAGE_BASE)/server:$(IMAGE_VERSION) \
--progress $(BUILDKIT_PROGRESS) \
--build-arg "KANIDM_BUILD_PROFILE=container_generic" \
--build-arg "KANIDM_FEATURES=" \
$(CONTAINER_BUILD_ARGS) .
@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/server:$(IMAGE_VERSION)
.PHONY: buildx/kanidm_tools
buildx/kanidm_tools: ## Build multiarch kanidm tool images and push to docker hub
buildx/kanidm_tools: vendor
buildx/kanidm_tools:
@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) \
--pull --push --platform $(IMAGE_ARCH) \
-f kanidm_tools/Dockerfile \
-t $(IMAGE_BASE)/tools:$(IMAGE_VERSION) \
--progress $(BUILDKIT_PROGRESS) \
--build-arg "KANIDM_BUILD_PROFILE=container_generic" \
--build-arg "KANIDM_FEATURES=" \
$(CONTAINER_BUILD_ARGS) .
@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/tools:$(IMAGE_VERSION)
.PHONY: buildx/radiusd
buildx/radiusd: ## Build multi-arch radius docker images and push to docker hub
@ -53,8 +54,8 @@ buildx/radiusd:
@$(CONTAINER_TOOL) buildx build $(CONTAINER_TOOL_ARGS) \
--pull --push --platform $(IMAGE_ARCH) \
-f kanidm_rlm_python/Dockerfile \
--progress $(BUILDKIT_PROGRESS) \
-t $(IMAGE_BASE)/radius:$(IMAGE_VERSION) .
@$(CONTAINER_TOOL) buildx imagetools $(CONTAINER_TOOL_ARGS) inspect $(IMAGE_BASE)/radius:$(IMAGE_VERSION)
.PHONY: buildx
buildx: buildx/kanidmd/x86_64_v3 buildx/kanidmd buildx/kanidm_tools buildx/radiusd
@ -114,11 +115,6 @@ install-tools: ## install kanidm_tools in your local environment
install-tools:
cd kanidm_tools && cargo install --path . --force
.PHONY: prep
prep:
cargo outdated -R
cargo audit
.PHONY: codespell
codespell:
codespell -c \
@ -215,6 +211,11 @@ docs/pykanidm/serve:
########################################################################
.PHONY: release/prep
prep:
cargo outdated -R
cargo audit
.PHONY: release/kanidm
release/kanidm: ## Build the Kanidm CLI - ensure you include the environment variable KANIDM_BUILD_PROFILE
cargo build -p kanidm_tools --bin kanidm --release
@ -241,7 +242,6 @@ release/kanidm-unixd:
--bin kanidm_cache_clear \
--bin kanidm_cache_invalidate
# cert things
.PHONY: cert/clean

12
RELEASE_NOTES.md
View file

@ -16,13 +16,13 @@ report it to our [issue tracker].
## 2023-02-01 - Kanidm 1.1.0-alpha11
This is the eleventh alpha series release of the Kanidm Identity Management project. Alpha releases are
to help get feedback and ideas from the community on how we can continue to make this project better
for a future supported release.
This is the eleventh alpha series release of the Kanidm Identity Management project. Alpha releases
are to help get feedback and ideas from the community on how we can continue to make this project
better for a future supported release.
The project is shaping up very nicely, and a beta will be coming soon! The main reason we haven't done
so yet is we haven't decided if we want to commit to the current API layout and freeze it yet. There
are still things we want to change there. Otherwise the server is stable and reliable.
The project is shaping up very nicely, and a beta will be coming soon! The main reason we haven't
done so yet is we haven't decided if we want to commit to the current API layout and freeze it yet.
There are still things we want to change there. Otherwise the server is stable and reliable.
### Release Highlights

42
kanidm_rlm_python/Dockerfile
View file

@ -1,10 +1,20 @@
FROM opensuse/tumbleweed:latest
ARG BASE_IMAGE=opensuse/tumbleweed:latest
FROM ${BASE_IMAGE} AS repos
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper mr -k repo-oss && \
zypper mr -k repo-update && \
zypper dup -y
# ======================
FROM repos
EXPOSE 1812 1813
# These all need to be on one line else the rpm cache ends
# up in the layers.
RUN zypper refresh --force && \
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper install -y \
freeradius-client \
freeradius-server \
@ -18,8 +28,7 @@ RUN zypper refresh --force && \
iproute2 \
iputils \
openssl \
curl && \
zypper clean
curl
ADD kanidm_rlm_python/mods-available/ /etc/raddb/mods-available/
COPY kanidm_rlm_python/sites-available/ /etc/raddb/sites-available/
@ -28,29 +37,30 @@ COPY kanidm_rlm_python/sites-available/ /etc/raddb/sites-available/
WORKDIR /etc/raddb
# Enable the python and cache module.
RUN ln -s /etc/raddb/mods-available/python3 /etc/raddb/mods-enabled/python3
RUN ln -s /etc/raddb/sites-available/check-eap-tls /etc/raddb/sites-enabled/check-eap-tls
RUN ln -s /etc/raddb/mods-available/python3 /etc/raddb/mods-enabled/python3 && \
ln -s /etc/raddb/sites-available/check-eap-tls /etc/raddb/sites-enabled/check-eap-tls
# disable auth via methods we don't support!
RUN rm /etc/raddb/mods-available/sql
RUN rm /etc/raddb/mods-enabled/{passwd,totp}
RUN rm /etc/raddb/mods-available/sql && \
rm /etc/raddb/mods-enabled/{passwd,totp}
# Allows the radiusd user to write to the directory
RUN chown -R radiusd: /etc/raddb
RUN chmod 775 /etc/raddb/certs
RUN chmod 640 /etc/raddb/clients.conf
RUN chown -R radiusd: /etc/raddb && \
chmod 775 /etc/raddb/certs && \
chmod 640 /etc/raddb/clients.conf
RUN mkdir -p /pkg/pykanidm/
COPY pykanidm/ /pkg/pykanidm/
# install the package and its dependencies
RUN python3 -m pip install --no-cache-dir --no-warn-script-location /pkg/pykanidm
# clean up after install
RUN rm -rf /pkg/*
RUN python3 -m pip install --no-cache-dir --no-warn-script-location /pkg/pykanidm && \
rm -rf /pkg/*
COPY kanidm_rlm_python/radius_entrypoint.py /radius_entrypoint.py
USER radiusd
ENV LD_PRELOAD=/usr/lib64/libpython3.so
ENV KANIDM_CONFIG_FILE="/data/kanidm"
COPY kanidm_rlm_python/radius_entrypoint.py /radius_entrypoint.py
USER radiusd
CMD [ "/usr/bin/python3", "/radius_entrypoint.py" ]

77
kanidm_tools/Dockerfile
View file

@ -1,19 +1,27 @@
# This builds the kanidm CLI tools
ARG BASE_IMAGE=opensuse/tumbleweed:latest
FROM ${BASE_IMAGE} AS repos
RUN zypper refresh --force
RUN zypper dup -y
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper mr -k repo-oss && \
zypper mr -k repo-non-oss && \
zypper mr -k repo-update && \
zypper dup -y
FROM repos AS builder
ARG KANIDM_FEATURES
ARG KANIDM_BUILD_PROFILE
ARG KANIDM_BUILD_OPTIONS=""
RUN echo Profile $KANIDM_BUILD_PROFILE
RUN echo Features $KANIDM_FEATURES
# Set the build profile
ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
RUN zypper install -y --no-recommends \
rustup wasm-pack \
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper install -y --no-recommends \
sccache \
cargo \
clang \
make automake autoconf \
libopenssl-3-devel \
@ -23,55 +31,44 @@ RUN zypper install -y --no-recommends \
rsync \
mold
RUN zypper clean -a
RUN rustup default stable
COPY . /usr/src/kanidm
RUN mkdir -p /usr/src/kanidm/.cargo
RUN cp /usr/src/kanidm/cargo_vendor_config /usr/src/kanidm/.cargo/config.toml
WORKDIR /usr/src/kanidm/
# Set the build profile
ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
# build the CLI
RUN if [ -z "${KANIDM_FEATURES}" ]; then \
cargo build -p kanidm_tools ${KANIDM_BUILD_OPTIONS} \
--target-dir="/usr/src/kanidm/target/" \
--release; \
cargo build -p kanidm-ipa-sync ${KANIDM_BUILD_OPTIONS} \
--target-dir="/usr/src/kanidm/target/" \
--release; \
else \
cargo build -p kanidm_tools ${KANIDM_BUILD_OPTIONS} \
--target-dir="/usr/src/kanidm/target/" \
--features="${KANIDM_FEATURES}" \
--release; \
cargo build -p kanidm-ipa-sync ${KANIDM_BUILD_OPTIONS} \
--target-dir="/usr/src/kanidm/target/" \
--features="${KANIDM_FEATURES}" \
--release; \
fi
RUN ls -al /usr/src/kanidm/target/release
RUN \
--mount=type=cache,id=cargo,target=/cargo \
--mount=type=cache,id=sccache,target=/sccache \
export CARGO_HOME=/cargo; \
export SCCACHE_DIR=/sccache; \
export RUSTC_WRAPPER=/usr/bin/sccache; \
export CC="/usr/bin/clang"; \
cargo build -p kanidm_tools ${KANIDM_BUILD_OPTIONS} \
--target-dir="/usr/src/kanidm/target/" \
--features="${KANIDM_FEATURES}" \
--release; \
cargo build -p kanidm-ipa-sync ${KANIDM_BUILD_OPTIONS} \
--target-dir="/usr/src/kanidm/target/" \
--features="${KANIDM_FEATURES}" \
--release; \
sccache -s
# == Construct the tools container
FROM repos
RUN zypper install -y timezone busybox-adduser openssl-3 && \
zypper clean -a
ENV RUST_BACKTRACE 1
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper install -y timezone busybox-adduser openssl-3
COPY --from=builder /usr/src/kanidm/target/release/kanidm /sbin/
COPY --from=builder /usr/src/kanidm/target/release/kanidm-ipa-sync /sbin/
RUN chmod +x /sbin/kanidm
RUN chmod +x /sbin/kanidm-ipa-sync
ENV RUST_BACKTRACE 1
RUN adduser -D -H kanidm
RUN mkdir /etc/kanidm && \
RUN adduser -D -H kanidm && \
mkdir /etc/kanidm && \
touch /etc/kanidm/config
USER kanidm

64
kanidmd/Dockerfile
View file

@ -1,8 +1,12 @@
# Build the main Kanidmd server
ARG BASE_IMAGE=opensuse/tumbleweed:latest
FROM ${BASE_IMAGE} AS repos
RUN zypper refresh --force
RUN zypper dup -y
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper mr -k repo-oss && \
zypper mr -k repo-non-oss && \
zypper mr -k repo-update && \
zypper dup -y
# ======================
FROM repos AS builder
@ -10,11 +14,15 @@ ARG KANIDM_FEATURES
ARG KANIDM_BUILD_PROFILE="container_generic"
ARG KANIDM_BUILD_OPTIONS=""
RUN echo Profile $KANIDM_BUILD_PROFILE
RUN echo Features $KANIDM_FEATURES
# Set the build profile
ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
RUN zypper install -y --no-recommends \
rustup \
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper install -y --no-recommends \
sccache \
cargo \
clang \
make automake autoconf \
libopenssl-3-devel pam-devel \
@ -23,17 +31,9 @@ RUN zypper install -y --no-recommends \
findutils \
which \
mold
# wasm-pack \
# lld
RUN zypper clean -a
RUN rustup default stable
COPY . /usr/src/kanidm
RUN mkdir -p /usr/src/kanidm/.cargo
RUN cp /usr/src/kanidm/cargo_vendor_config /usr/src/kanidm/.cargo/config.toml
# ======================
# WORKDIR /usr/src/kanidm/kanidmd_web_ui
@ -45,37 +45,31 @@ RUN cp /usr/src/kanidm/cargo_vendor_config /usr/src/kanidm/.cargo/config.toml
WORKDIR /usr/src/kanidm/kanidmd/daemon
# Set the build profile
ENV KANIDM_BUILD_PROFILE=${KANIDM_BUILD_PROFILE:-container_generic}
ENV RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/bin/ld.mold"
# Exports don't persist through RUN statements.
RUN export CC="/usr/bin/clang"; \
if [ -z "${KANIDM_FEATURES}" ]; then \
cargo build -p daemon ${KANIDM_BUILD_OPTIONS} \
--locked --offline \
--target-dir="/usr/src/kanidm/target/" \
--release; \
else \
cargo build -p daemon ${KANIDM_BUILD_OPTIONS} \
--locked --offline \
--target-dir="/usr/src/kanidm/target/" \
--features="${KANIDM_FEATURES}" \
--release; \
fi
RUN ls -al /usr/src/kanidm/target/release
RUN \
--mount=type=cache,id=cargo,target=/cargo \
--mount=type=cache,id=sccache,target=/sccache \
export CARGO_HOME=/cargo; \
export SCCACHE_DIR=/sccache; \
export RUSTC_WRAPPER=/usr/bin/sccache; \
export CC="/usr/bin/clang"; \
cargo build -p daemon ${KANIDM_BUILD_OPTIONS} \
--target-dir="/usr/src/kanidm/target/" \
--features="${KANIDM_FEATURES}" \
--release; \
sccache -s
# ======================
FROM repos
RUN zypper install -y \
RUN \
--mount=type=cache,id=zypp,target=/var/cache/zypp \
zypper install -y \
timezone \
openssl-3 \
sqlite3 \
pam
RUN zypper clean -a
COPY --from=builder /usr/src/kanidm/target/release/kanidmd /sbin/
COPY --from=builder /usr/src/kanidm/kanidmd_web_ui/pkg /pkg

6
kanidmd/lib/src/be/mod.rs
View file

@ -1714,9 +1714,9 @@ impl Backend {
idxkeys: Vec<IdxKey>,
vacuum: bool,
) -> Result<Self, OperationError> {
debug!("DB tickets -> {:?}", cfg.pool_size);
debug!("Profile -> {}", env!("KANIDM_PROFILE_NAME"));
debug!("CPU Flags -> {}", env!("KANIDM_CPU_FLAGS"));
info!("DB tickets -> {:?}", cfg.pool_size);
info!("Profile -> {}", env!("KANIDM_PROFILE_NAME"));
info!("CPU Flags -> {}", env!("KANIDM_CPU_FLAGS"));
// If in memory, reduce pool to 1
if cfg.path.is_empty() {

4
profiles/container_generic.toml
View file

@ -1,3 +1,3 @@
web_ui_pkg_path = "/pkg"
# Valid options are none, native, x86_64_v1, x86_64_v3
cpu_flags = "none"
# Don't set the cpu_flags to autodetect for this platform
# cpu_flags = "none"

2
profiles/container_x86_64_v3.toml
View file

@ -1,3 +1,3 @@
web_ui_pkg_path = "/pkg"
# Valid options are none, native, x86_64, x86_64_v3
# Define this to override the arch.
cpu_flags = "x86_64_v3"

2
profiles/developer.toml
View file

@ -1,3 +1,3 @@
web_ui_pkg_path = "../../kanidmd_web_ui/pkg"
# Valid options are none, native, x86_64, x86_64_v3
# Set to native for developer machines.
cpu_flags = "native"

3
profiles/release_suse_generic.toml
View file

@ -1,2 +1,3 @@
web_ui_pkg_path = "/usr/share/kanidm/ui/pkg"
cpu_flags = "none"
# Don't set the value for autodetect
# cpu_flags = "none"

2
profiles/release_suse_x86_64.toml
View file

@ -1,2 +0,0 @@
web_ui_pkg_path = "/usr/share/kanidm/ui/pkg"
cpu_flags = "x86_64_v1"

30
profiles/src/lib.rs
View file

@ -7,16 +7,30 @@ use serde::Deserialize;
enum CpuOptLevel {
none,
native,
x86_64_v1,
neon_v8,
x86_64_v2,
x86_64_v3,
}
impl Default for CpuOptLevel {
fn default() -> Self {
if cfg!(target_arch = "x86_64") {
CpuOptLevel::x86_64_v2
} else if cfg!(target_arch = "aarch64") {
CpuOptLevel::neon_v8
} else {
CpuOptLevel::none
}
}
}
impl std::fmt::Display for CpuOptLevel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self {
CpuOptLevel::none => write!(f, "none"),
CpuOptLevel::native => write!(f, "native"),
CpuOptLevel::x86_64_v1 => write!(f, "x86_64_v1"),
CpuOptLevel::neon_v8 => write!(f, "neon_v8"),
CpuOptLevel::x86_64_v2 => write!(f, "x86_64_v2"),
CpuOptLevel::x86_64_v3 => write!(f, "x86_64_v3"),
}
}
@ -25,6 +39,7 @@ impl std::fmt::Display for CpuOptLevel {
#[derive(Debug, Deserialize)]
struct ProfileConfig {
web_ui_pkg_path: String,
#[serde(default)]
cpu_flags: CpuOptLevel,
}
@ -43,17 +58,12 @@ pub fn apply_profile() {
let profile_cfg: ProfileConfig = toml::from_slice(&data)
.unwrap_or_else(|_| panic!("Failed to parse profile - {} - {}", profile, contents));
/*
* x86-64: CMOV, CMPXCHG8B, FPU, FXSR, MMX, FXSR, SCE, SSE, SSE2
* x86-64-v2: (close to Nehalem) CMPXCHG16B, LAHF-SAHF, POPCNT, SSE3, SSE4.1, SSE4.2, SSSE3
* x86-64-v3: (close to Haswell) AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE, XSAVE
*/
match profile_cfg.cpu_flags {
CpuOptLevel::none => {}
CpuOptLevel::native => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-cpu=native"),
CpuOptLevel::x86_64_v1 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-feature=+cmov,+cx8,+fxsr,+mmx,+sse,+sse2"),
CpuOptLevel::x86_64_v3 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-feature=+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+cx16,+sahf,+popcnt,+sse3,+sse4.1,+sse4.2,+avx,+avx2,+bmi,+bmi2,+f16c,+fma,+lzcnt,+movbe,+xsave"),
CpuOptLevel::neon_v8 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-features=+neon,+fp-armv8"),
CpuOptLevel::x86_64_v2 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-cpu=x86-64-v2"),
CpuOptLevel::x86_64_v3 => println!("cargo:rustc-env=RUSTFLAGS=-Ctarget-cpu=x86-64-v3"),
}
println!("cargo:rustc-env=KANIDM_PROFILE_NAME={}", profile);
println!("cargo:rustc-env=KANIDM_CPU_FLAGS={}", profile_cfg.cpu_flags);

4
project_docs/RELEASE_CHECKLIST.md
View file

@ -47,8 +47,8 @@ cargo install cargo-outdated
- [ ] git push origin 1.1.0-alpha.x
- [ ] git push origin 1.1.0-alpha.x --tags
- [ ] github -> create new release based on tag (not branch)
- use tag because then tools will get the tag + patches we apply.
- [ ] github -> create new release based on tag (not branch) - use tag because then tools will get
the tag + patches we apply.
### Cargo publish