From bca2fbcf4ed4c4d2630baa6efffd9be8b006c37a Mon Sep 17 00:00:00 2001 From: James Hodgkinson Date: Mon, 27 Nov 2023 16:30:21 +1000 Subject: [PATCH] Unix crossbuild scripts and docs (#2326) * can build now with cargo cross --- .github/workflows/debian_package_kanidm.yml | 2 +- platform/crossbuild/README.md | 40 +++++++++++++++++++++ platform/crossbuild/build.sh | 40 +++++++++++++++++++++ platform/crossbuild/ubuntu-20.04/Cross.toml | 18 ++++++++++ platform/crossbuild/ubuntu-20.04/Dockerfile | 14 ++++++++ platform/crossbuild/ubuntu-22.04/Cross.toml | 17 +++++++++ platform/crossbuild/ubuntu-22.04/Dockerfile | 14 ++++++++ platform/debian/Makefile | 9 +++-- platform/debian/build_kanidm.sh | 11 ++++++ platform/debian/kanidm-unixd/rules | 1 + unix_integration/Cargo.toml | 10 +++++- 11 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 platform/crossbuild/README.md create mode 100755 platform/crossbuild/build.sh create mode 100644 platform/crossbuild/ubuntu-20.04/Cross.toml create mode 100644 platform/crossbuild/ubuntu-20.04/Dockerfile create mode 100644 platform/crossbuild/ubuntu-22.04/Cross.toml create mode 100644 platform/crossbuild/ubuntu-22.04/Dockerfile diff --git a/.github/workflows/debian_package_kanidm.yml b/.github/workflows/debian_package_kanidm.yml index 6b1a3d672..a5698830f 100644 --- a/.github/workflows/debian_package_kanidm.yml +++ b/.github/workflows/debian_package_kanidm.yml @@ -74,4 +74,4 @@ jobs: automatic_release_tag: "debs" prerelease: true title: ".deb Packages" - files: "*.deb" + files: "*.deb" \ No newline at end of file diff --git a/platform/crossbuild/README.md b/platform/crossbuild/README.md new file mode 100644 index 000000000..0a382871b --- /dev/null +++ b/platform/crossbuild/README.md @@ -0,0 +1,40 @@ +# Cross-building things using cargo cross + +Here be dragons. + +1. Get a drink. You'l need it. +2. Install [cargo-cross](https://github.com/cross-rs/cross) +3. Drink the drink. + +## Building Ubuntu 20.04 things + +Make sure you're including `--release` because reasons. + +```shell +CROSS_CONFIG=platform/crossbuild/ubuntu-20.04/Cross.toml \ + cross build --target aarch64-unknown-linux-gnu \ + --bin kanidm_unixd \ + --bin kanidm_unixd_tasks \ + --bin kanidm_ssh_authorizedkeys \ + --bin kanidm-unix \ + --release +``` + +Things will end up in `./target/aarch64-unknown-linux-gnu/release/` + +## Building Ubuntu 22.04 things + +Make sure you're including `--release` because reasons. + +```shell +CROSS_CONFIG=platform/crossbuild/ubuntu-22.04/Cross.toml \ + cross build --target aarch64-unknown-linux-gnu \ + --bin kanidm_unixd \ + --bin kanidm_unixd_tasks \ + --bin kanidm_ssh_authorizedkeys \ + --bin kanidm-unix \ + --release + +``` + +Things will end up in `./target/aarch64-unknown-linux-gnu/release/` diff --git a/platform/crossbuild/build.sh b/platform/crossbuild/build.sh new file mode 100755 index 000000000..ca8647871 --- /dev/null +++ b/platform/crossbuild/build.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +if [ -z "$1" ]; then + echo "Usage: $0 target_os" + if [ -d ./platform ]; then + echo "Options:" + find platform/crossbuild -type d -maxdepth 1 -mindepth 1 | awk -F'/' '{print $NF}' | sort + fi + exit 1 +fi + +if [ ! -d "platform/crossbuild/$1" ]; then + echo "Could not find platform/crossbuild/$1" + exit 1 +fi + +TARGET_DIR="./target/$1" + +echo "Recreating then building to ${TARGET_DIR}" +rm -rf "${TARGET_DIR}" +mkdir -p "${TARGET_DIR}" + +CROSS_CONFIG="platform/crossbuild/${1}/Cross.toml" \ + cross build --target aarch64-unknown-linux-gnu \ + --bin kanidm_unixd \ + --bin kanidm_unixd_tasks \ + --bin kanidm_ssh_authorizedkeys \ + --bin kanidm-unix \ + --release + +find "./target/aarch64-unknown-linux-gnu/release/" -maxdepth 1 \ + -type f -not -name '*.d' \ + -name 'kanidm*' \ + -exec mv "{}" "${TARGET_DIR}/" \; +# find "${TARGET_DIR}" -name '*.d' -delete + +echo "Contents of ${TARGET_DIR}" +find "${TARGET_DIR}" -type f diff --git a/platform/crossbuild/ubuntu-20.04/Cross.toml b/platform/crossbuild/ubuntu-20.04/Cross.toml new file mode 100644 index 000000000..b0e37ef65 --- /dev/null +++ b/platform/crossbuild/ubuntu-20.04/Cross.toml @@ -0,0 +1,18 @@ +[build.env] + + +[target.aarch64-unknown-linux-gnu] +dockerfile = "platform/crossbuild/ubuntu-20.04/Dockerfile" +pre-build = [ + "TZ=UTC ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone", + + "apt-get update && apt-get install --assume-yes libssl-dev build-essential", + + # "sed 's/^deb http/deb [arch=amd64] http/' -i '/etc/apt/sources.list'", + # "echo 'deb [arch=arm64] http://au.archive.ubuntu.com/pub/ubuntu/ports jammy main restricted universe multiverse' >> /etc/apt/sources.list", + # "echo 'deb [arch=arm64] http://au.archive.ubuntu.com/pub/ubuntu/ports jammy-updates main restricted universe multiverse' >> /etc/apt/sources.list", + # "echo 'deb [arch=arm64] http://au.archive.ubuntu.com/pub/ubuntu/ports jammy-backports main restricted universe multiverse' >> /etc/apt/sources.list",do + + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH libpam0g-dev:$CROSS_DEB_ARCH libudev-dev:$CROSS_DEB_ARCH pkg-config:$CROSS_DEB_ARCH", +] diff --git a/platform/crossbuild/ubuntu-20.04/Dockerfile b/platform/crossbuild/ubuntu-20.04/Dockerfile new file mode 100644 index 000000000..f70d7910d --- /dev/null +++ b/platform/crossbuild/ubuntu-20.04/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:20.04 +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install --assume-yes --no-install-recommends \ + g++-aarch64-linux-gnu \ + libc6-dev-arm64-cross + +ENV CROSS_TOOLCHAIN_PREFIX=aarch64-linux-gnu- +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc \ + AR_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"ar \ + CC_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"gcc \ + CXX_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"g++ \ + RUST_TEST_THREADS=1 \ + PKG_CONFIG_PATH="/usr/lib/aarch64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}" \ No newline at end of file diff --git a/platform/crossbuild/ubuntu-22.04/Cross.toml b/platform/crossbuild/ubuntu-22.04/Cross.toml new file mode 100644 index 000000000..8fa52cbab --- /dev/null +++ b/platform/crossbuild/ubuntu-22.04/Cross.toml @@ -0,0 +1,17 @@ +[build.env] + +[target.aarch64-unknown-linux-gnu] +dockerfile = "platform/crossbuild/ubuntu-22.04/Dockerfile" +pre-build = [ + "TZ=UTC ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone", + + "apt-get update && apt-get install --assume-yes rsync libssl-dev build-essential", + + "sed 's/^deb http/deb [arch=amd64] http/' -i '/etc/apt/sources.list'", + "echo 'deb [arch=arm64] http://au.archive.ubuntu.com/pub/ubuntu/ports jammy main restricted universe multiverse' >> /etc/apt/sources.list", + "echo 'deb [arch=arm64] http://au.archive.ubuntu.com/pub/ubuntu/ports jammy-updates main restricted universe multiverse' >> /etc/apt/sources.list", + "echo 'deb [arch=arm64] http://au.archive.ubuntu.com/pub/ubuntu/ports jammy-backports main restricted universe multiverse' >> /etc/apt/sources.list", + + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH libpam0g-dev:$CROSS_DEB_ARCH libudev-dev:$CROSS_DEB_ARCH pkg-config:$CROSS_DEB_ARCH", +] diff --git a/platform/crossbuild/ubuntu-22.04/Dockerfile b/platform/crossbuild/ubuntu-22.04/Dockerfile new file mode 100644 index 000000000..a8079f894 --- /dev/null +++ b/platform/crossbuild/ubuntu-22.04/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:22.04 +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install --assume-yes --no-install-recommends \ + g++-aarch64-linux-gnu \ + libc6-dev-arm64-cross + +ENV CROSS_TOOLCHAIN_PREFIX=aarch64-linux-gnu- +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc \ + AR_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"ar \ + CC_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"gcc \ + CXX_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"g++ \ + RUST_TEST_THREADS=1 \ + PKG_CONFIG_PATH="/usr/lib/aarch64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}" \ No newline at end of file diff --git a/platform/debian/Makefile b/platform/debian/Makefile index e2fedeb7e..6b5b60ada 100644 --- a/platform/debian/Makefile +++ b/platform/debian/Makefile @@ -1,3 +1,6 @@ +ARCH ?= $(dpkg --print-architecture) + + .DEFAULT: help .PHONY: help @@ -7,17 +10,17 @@ help: .PHONY: debs/kanidm debs/kanidm: ## Build the Kanidm CLI package - make sure you set the environment variable KANIDM_BUILD_PROFILE debs/kanidm: - bash ./platform/debian/build_kanidm.sh kanidm + ARCH=$(ARCH) bash ./platform/debian/build_kanidm.sh kanidm .PHONY: debs/kanidmd debs/kanidmd: ## Build the Kanidmd package - make sure you set the environment variable KANIDM_BUILD_PROFILE debs/kanidmd: - bash ./platform/debian/build_kanidm.sh kanidmd + ARCH=$(ARCH) bash ./platform/debian/build_kanidm.sh kanidmd .PHONY: debs/kanidm-unixd debs/kanidm-unixd: ## Build the Kanidm UNIX tools package (PAM/NSS, unixd and related tools) - make sure you set the environment variable KANIDM_BUILD_PROFILE debs/kanidm-unixd: - bash ./platform/debian/build_kanidm.sh kanidm-unixd + ARCH=$(ARCH) bash ./platform/debian/build_kanidm.sh kanidm-unixd .PHONY: debs/all debs/all: ## Build all the .deb packages - make sure you set the environment variable KANIDM_BUILD_PROFILE diff --git a/platform/debian/build_kanidm.sh b/platform/debian/build_kanidm.sh index 0caac9817..3bc247587 100755 --- a/platform/debian/build_kanidm.sh +++ b/platform/debian/build_kanidm.sh @@ -4,6 +4,17 @@ set -e +if [ -z "${ARCH}" ]; then + ARCH="$(dpkg --print-architecture)" +fi + +if [ "${ARCH}" -ne "$(dpkg --print-architecture)" ]; then + echo "${ARCH} -ne $(dpkg --print-architecture), cross-compiling!" + export PKG_CONFIG_ALLOW_CROSS=1 + +fi + + if [ -z "$1" ]; then PACKAGE="kanidm" else diff --git a/platform/debian/kanidm-unixd/rules b/platform/debian/kanidm-unixd/rules index 39cef815f..f07858d5b 100755 --- a/platform/debian/kanidm-unixd/rules +++ b/platform/debian/kanidm-unixd/rules @@ -25,6 +25,7 @@ override_dh_auto_clean: override_dh_autoreconf: override_dh_auto_build: + # this runs "make -- release/${PACKAGE} release/kanidm-ssh" KANIDM_BUILD_PROFILE=release_suse_generic dh_auto_build -- release/${PACKAGE} release/kanidm-ssh override_dh_auto_test: diff --git a/unix_integration/Cargo.toml b/unix_integration/Cargo.toml index 7000b2346..3dd24f214 100644 --- a/unix_integration/Cargo.toml +++ b/unix_integration/Cargo.toml @@ -77,7 +77,15 @@ serde_json = { workspace = true } sketching = { workspace = true } toml = { workspace = true } -tokio = { workspace = true, features = ["rt", "fs", "macros", "sync", "time", "net", "io-util"] } +tokio = { workspace = true, features = [ + "rt", + "fs", + "macros", + "sync", + "time", + "net", + "io-util", +] } tokio-util = { workspace = true, features = ["codec"] } tracing = { workspace = true } uuid = { workspace = true }