k3s: v1.22.3+k3s1 -> 1.23.3+k3s1

This PR also reworks the packaging of k3s significantly.

Notably, it makes the following changes:

1. buildGoModule instead of buildGoPackage + mkDerivation (for the final one)
2. Merges the final two derivations (k3s user binary + wrapper script stuff) into one
3. Reduces the amount of patching & usage of upstream packaging scripts

And of course updates to 1.23

I'm not really sure yet if this is significantly better than the
previous packaging model, but it at least seems a little nicer I guess?
buildGoModule is nice, even if we have to hack around it quite a bit to
get a 'go generate' in there.
This commit is contained in:
Euan Kemp 2022-02-01 01:15:56 -08:00
parent 9f697d60e4
commit 7a7335d43e
5 changed files with 219 additions and 203 deletions

View file

@ -5,9 +5,11 @@
, iptables
, iproute2
, bridge-utils
, btrfs-progs
, conntrack-tools
, buildGoPackage
, buildGoModule
, runc
, rsync
, kmod
, libseccomp
, pkg-config
@ -18,6 +20,7 @@
, fetchzip
, fetchgit
, zstd
, yq-go
, nixosTests
}:
@ -43,19 +46,31 @@ with lib;
# Those pieces of software we entirely ignore upstream's handling of, and just
# make sure they're in the path if desired.
let
k3sVersion = "1.22.3+k3s1"; # k3s git tag
k3sCommit = "61a2aab25eeb97c26fa3f2b177e4355a7654c991"; # k3s git commit at the above version
k3sRepoSha256 = "0lz5hr3c86gxm9w5jy3g26n6a26m8k0y559hv6220rsi709j7ma9";
k3sVersion = "1.23.3+k3s1"; # k3s git tag
k3sCommit = "6f4217a3405d16a1a51bbb40872d7dcb87207bb9"; # k3s git commit at the above version
k3sRepoSha256 = "sha256-0dRusG1vL+1KbmViIUNCZK1b+FEgV6otcVUyFonHmm4=";
traefikChartVersion = "10.3.0"; # taken from ./manifests/traefik.yaml at spec.version
traefikChartSha256 = "0y6wr64xp7bgx24kqil0x6myr3pnfrg8rw0d1h5zd2n5a8nfd73f";
# taken from ./manifests/traefik.yaml, extracted from '.spec.chart' https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/download#L9
# The 'patch' and 'minor' versions are currently hardcoded as single digits only, so ignore the trailing two digits. Weird, I know.
traefikChartVersion = "10.9.1";
traefikChartSha256 = "sha256-XM1DLofU1zEEFeB5bNQ7cgv102gXsToPP7SFh87QuGQ=";
k3sRootVersion = "0.9.1"; # taken from ./scripts/download at ROOT_VERSION
# taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47
k3sRootVersion = "0.9.1";
k3sRootSha256 = "0r2cj4l50cxkrvszpzxfk36lvbjf9vcmp6d5lvxg8qsah8lki3x8";
k3sCNIVersion = "0.9.1-k3s1"; # taken from ./scripts/version.sh at VERSION_CNIPLUGINS
# taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45
k3sCNIVersion = "0.9.1-k3s1";
k3sCNISha256 = "1327vmfph7b8i14q05c2xdfzk60caflg1zhycx0mrf3d59f4zsz5";
# taken from go.mod, the 'github.com/containerd/containerd' line
# run `grep github.com/containerd/containerd go.mod | head -n1 | awk '{print $4}'`
containerdVersion = "v1.5.9-k3s1";
containerdSha256 = "sha256-7xlhBA6KuwFlw+jyThygv4Ow9F3xjjIUtS6x8YHwjic=";
# run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag
criCtlVersion = "v1.22.0-k3s1";
baseMeta = {
description = "A lightweight Kubernetes distribution";
license = licenses.asl20;
@ -64,6 +79,25 @@ let
platforms = platforms.linux;
};
# https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40
versionldflags = [
"-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}"
"-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}"
"-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}"
"-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}"
"-X k8s.io/client-go/pkg/version.gitTreeState=clean"
"-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z"
"-X k8s.io/component-base/version.gitVersion=v${k3sVersion}"
"-X k8s.io/component-base/version.gitCommit=${k3sCommit}"
"-X k8s.io/component-base/version.gitTreeState=clean"
"-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z"
"-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=${criCtlVersion}"
"-X github.com/containerd/containerd/version.Version=${containerdVersion}"
"-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd"
"-X github.com/containerd/containerd/version.Version=${containerdVersion}"
"-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd"
];
# bundled into the k3s binary
traefikChart = fetchurl {
url = "https://helm.traefik.io/traefik/traefik-${traefikChartVersion}.tgz";
@ -84,11 +118,11 @@ let
sha256 = k3sRootSha256;
stripRoot = false;
};
k3sPlugins = buildGoPackage rec {
name = "k3s-cni-plugins";
k3sCNIPlugins = buildGoModule rec {
pname = "k3s-cni-plugins";
version = k3sCNIVersion;
vendorSha256 = null;
goPackagePath = "github.com/containernetworking/plugins";
subPackages = [ "." ];
src = fetchFromGitHub {
@ -98,6 +132,10 @@ let
sha256 = k3sCNISha256;
};
postInstall = ''
mv $out/bin/plugins $out/bin/cni
'';
meta = baseMeta // {
description = "CNI plugins, as patched by rancher for k3s";
};
@ -124,50 +162,38 @@ let
# Then, we bundle those binaries into our thick k3s binary and use that as
# the final single output.
# This approach was chosen because it ensures the bundled binaries all are
# correctly built to run with nix (we can lean on the existing buildGoPackage
# correctly built to run with nix (we can lean on the existing buildGoModule
# stuff), and we can again lean on that tooling for the final k3s binary too.
# Other alternatives would be to manually run the
# strip/patchelf/remove-references step ourselves in the installPhase of the
# derivation when we've built all the binaries, but haven't bundled them in
# with generated bindata yet.
k3sBuildStage1 = buildGoPackage rec {
name = "k3s-build-1";
k3sServer = buildGoModule rec {
pname = "k3s-server";
version = k3sVersion;
goPackagePath = "github.com/rancher/k3s";
src = k3sRepo;
# Patch build scripts so that we can use them.
# This makes things more dynamically linked (because nix can deal with
# dynamically linked dependencies just fine), removes the upload at the
# end, and skips building runc + cni, since we have our own derivations for
# those.
patches = [ ./patches/0002-Add-nixpkgs-patches.patch ];
vendorSha256 = "sha256-9+2k/ipAOhc8JJU+L2dwaM01Dkw+0xyrF5kt6mL19G0=";
nativeBuildInputs = [ pkg-config ];
buildInputs = [ libseccomp ];
# Versioning info for build script
DRONE_TAG = "v${version}";
DRONE_COMMIT = k3sCommit;
buildPhase = ''
pushd go/src/${goPackagePath}
patchShebangs ./scripts/build ./scripts/version.sh
mkdir -p bin
./scripts/build
popd
'';
installPhase = ''
pushd go/src/${goPackagePath}
mkdir -p "$out/bin"
install -m 0755 -t "$out/bin" ./bin/*
subPackages = [ "cmd/server" ];
ldflags = versionldflags;
# create the multicall symlinks for k3s
postInstall = ''
mv $out/bin/server $out/bin/k3s
pushd $out
# taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113
ln -s k3s ./bin/k3s-agent
ln -s k3s ./bin/k3s-server
ln -s k3s ./bin/k3s-etcd-snapshot
ln -s k3s ./bin/k3s-secrets-encrypt
ln -s k3s ./bin/k3s-certificate
ln -s k3s ./bin/kubectl
ln -s k3s ./bin/crictl
ln -s k3s ./bin/ctr
popd
'';
@ -175,76 +201,33 @@ let
description = "The various binaries that get packaged into the final k3s binary";
};
};
k3sBin = buildGoPackage rec {
name = "k3s-bin";
k3sContainerd = buildGoModule {
pname = "k3s-containerd";
version = k3sVersion;
goPackagePath = "github.com/rancher/k3s";
src = k3sRepo;
# See the above comment in k3sBuildStage1
patches = [ ./patches/0002-Add-nixpkgs-patches.patch ];
nativeBuildInputs = [ pkg-config zstd ];
# These dependencies are embedded as compressed files in k3s at runtime.
# Propagate them to avoid broken runtime references to libraries.
propagatedBuildInputs = [ k3sPlugins k3sBuildStage1 runc ];
# k3s appends a suffix to the final distribution binary for some arches
archSuffix =
if stdenv.hostPlatform.system == "x86_64-linux" then ""
else if stdenv.hostPlatform.system == "aarch64-linux" then "-arm64"
else throw "k3s isn't being built for ${stdenv.hostPlatform.system} yet.";
DRONE_TAG = "v${version}";
DRONE_COMMIT = k3sCommit;
# In order to build the thick k3s binary (which is what
# ./scripts/package-cli does), we need to get all the binaries that script
# expects in place.
buildPhase = ''
pushd go/src/${goPackagePath}
patchShebangs ./scripts/build ./scripts/version.sh ./scripts/package-cli
mkdir -p bin
install -m 0755 -t ./bin ${k3sBuildStage1}/bin/*
install -m 0755 -T "${k3sPlugins}/bin/plugins" ./bin/cni
# Note: use the already-nixpkgs-bundled k3s rather than the one bundled
# in k3s because the k3s one is completely unmodified from upstream
# (unlike containerd, cni, etc)
install -m 0755 -T "${runc}/bin/runc" ./bin/runc
cp -R "${k3sRoot}/etc" ./etc
mkdir -p "build/static/charts"
cp "${traefikChart}" "build/static/charts/traefik-${traefikChartVersion}.tgz"
./scripts/package-cli
popd
'';
installPhase = ''
pushd go/src/${goPackagePath}
mkdir -p "$out/bin"
install -m 0755 -T ./dist/artifacts/k3s${archSuffix} "$out/bin/k3s"
popd
'';
meta = baseMeta // {
description = "The k3s go binary which is used by the final wrapped output below";
src = fetchFromGitHub {
owner = "k3s-io";
repo = "containerd";
rev = containerdVersion;
sha256 = containerdSha256;
};
vendorSha256 = null;
buildInputs = [ btrfs-progs ];
subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ];
ldflags = versionldflags;
};
in
stdenv.mkDerivation rec {
buildGoModule rec {
pname = "k3s";
version = k3sVersion;
# `src` here is a workaround for the updateScript bot. It couldn't be empty.
src = builtins.filterSource (path: type: false) ./.;
src = k3sRepo;
proxyVendor = true;
vendorSha256 = "sha256-8Yp9csyRNSYi9wo8E8mF8cu92wG1t3l18wJ8Y4L7HEA=";
patches = [
./patches/0001-scrips-download-strip-downloading-just-package-CRD.patch
./patches/0002-Don-t-build-a-static-binary-in-package-cli.patch
];
# Important utilities used by the kubelet, see
# https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494
@ -260,32 +243,68 @@ stdenv.mkDerivation rec {
conntrack-tools
];
buildInputs = [
k3sBin
] ++ k3sRuntimeDeps;
buildInputs = k3sRuntimeDeps;
nativeBuildInputs = [ makeWrapper ];
nativeBuildInputs = [
makeWrapper
rsync
yq-go
zstd
];
unpackPhase = "true";
# embedded in the final k3s cli
propagatedBuildInputs = [
k3sCNIPlugins
k3sContainerd
k3sServer
runc
];
# We override most of buildPhase due to peculiarities in k3s's build.
# Specifically, it has a 'go generate' which runs part of the package. See
# this comment:
# https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694
# So, why do we use buildGoModule at all? For the `vendorSha256` / `go mod download` stuff primarily.
buildPhase = ''
patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload
# copy needed 'go generate' inputs into place
mkdir -p ./bin/aux
rsync -a --no-perms ${k3sServer}/bin/ ./bin/
ln -vsf ${runc}/bin/runc ./bin/runc
ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni
ln -vsf ${k3sContainerd}/bin/* ./bin/
rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/
mkdir -p ./build/static/charts
# Note, upstream's chart has a 00 suffix. This seems to not matter though, so we're ignoring that naming detail.
export TRAEFIK_CHART_FILE=${traefikChart}
# place the traefik chart using their code since it's complicated
# We trim the actual download, see patches
./scripts/download
export ARCH=$GOARCH
export DRONE_TAG="v${k3sVersion}"
export DRONE_COMMIT="${k3sCommit}"
# use ./scripts/package-cli to run 'go generate' + 'go build'
./scripts/package-cli
mkdir -p $out/bin
'';
# Otherwise it depends on 'getGoDirs', which is normally set in buildPhase
doCheck = false;
# And, one final derivation (you thought the last one was it, right?)
# We got the binary we wanted above, but it doesn't have all the runtime
# dependencies k8s wants, including mount utilities for kubelet, networking
# tools for cni/kubelet stuff, etc
# Use a wrapper script to reference all the binaries that k3s tries to
# execute, but that we didn't bundle with it.
installPhase = ''
runHook preInstall
mkdir -p "$out/bin"
makeWrapper ${k3sBin}/bin/k3s "$out/bin/k3s" \
# wildcard to match the arm64 build too
install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s
wrapProgram $out/bin/k3s \
--prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \
--prefix PATH : "$out/bin"
runHook postInstall
'';
doInstallCheck = true;
installCheckPhase = ''
$out/bin/k3s --version | grep v${k3sVersion} > /dev/null
$out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null
'';
passthru.updateScript = ./update.sh;

View file

@ -0,0 +1,41 @@
From 6f53bd36a40da4c71486e3b79f6e32d53d6eea5d Mon Sep 17 00:00:00 2001
From: Euan Kemp <euank@euank.com>
Date: Thu, 3 Feb 2022 23:50:40 -0800
Subject: [PATCH 2/2] scrips/download: strip downloading, just package CRD
The CRD packaging is a complicated set of commands, so let's reuse it.
---
scripts/download | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
diff --git a/scripts/download b/scripts/download
index 5effc0562a..82361803ee 100755
--- a/scripts/download
+++ b/scripts/download
@@ -24,12 +24,6 @@ rm -rf ${CONTAINERD_DIR}
mkdir -p ${CHARTS_DIR}
mkdir -p ${DATA_DIR}
-curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf - --exclude=bin/socat
-
-git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
-
-git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://github.com/k3s-io/containerd ${CONTAINERD_DIR}
-
setup_tmp() {
TMP_DIR=$(mktemp -d --tmpdir=${CHARTS_DIR})
cleanup() {
@@ -44,8 +38,8 @@ setup_tmp() {
download_and_package_traefik () {
echo "Downloading Traefik Helm chart from ${TRAEFIK_URL}"
- curl -sfL ${TRAEFIK_URL} -o ${TMP_DIR}/${TRAEFIK_FILE}
- code=$?
+ # nixpkgs: copy in our known traefik chart instead
+ cp $TRAEFIK_CHART_FILE ${TMP_DIR}/${TRAEFIK_FILE}
if [ $code -ne 0 ]; then
echo "Error: Failed to download Traefik Helm chart!"
--
2.34.1

View file

@ -1,81 +0,0 @@
-Subject: [PATCH 2/2] Add nixpkgs patches
-Original patch by: Euan Kemp <euank@euank.com>
-Adapted by: superherointj
-
-This patch allows us to re-use upstream build scripts when building for nix.
----
- 2 files changed:
- scripts/build
- scripts/package-cli
-
diff --git a/scripts/build b/scripts/build
index 2f3d1dc496..4f4e5aa897 100755
--- a/scripts/build
+++ b/scripts/build
@@ -12,7 +12,8 @@ PKG_CONTAINERD="github.com/containerd/containerd"
PKG_K3S_CONTAINERD="github.com/k3s-io/containerd"
PKG_CRICTL="github.com/kubernetes-sigs/cri-tools"
-buildDate=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+# nixpkgs: deterministic build date
+buildDate="$(date -d "$(git log -1 --format=%ai)" -u "+%Y-%m-%dT%H:%M:%SZ")"
VENDOR_PREFIX="${PKG}/vendor/"
VERSIONFLAGS="
@@ -89,17 +90,7 @@ cleanup() {
}
INSTALLBIN=$(pwd)/bin
-if [ ! -x ${INSTALLBIN}/cni ]; then
-(
- echo Building cni
- TMPDIR=$(mktemp -d)
- trap cleanup EXIT
- WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
- git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
- cd $WORKDIR
- GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni
-)
-fi
+# nixpkgs: skip building cni, we build it separately
# echo Building agent
# CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go
echo Building server
@@ -116,10 +107,7 @@ ln -s containerd ./bin/ctr
#CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/ctr ./cmd/ctr/main.go
# echo Building containerd
# CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/
-echo Building runc
-rm -f ./build/src/github.com/opencontainers/runc/runc
-make GOPATH=$(pwd)/build EXTRA_LDFLAGS="-w -s" BUILDTAGS="$RUNC_TAGS" -C ./build/src/github.com/opencontainers/runc $RUNC_STATIC
-cp -f ./build/src/github.com/opencontainers/runc/runc ./bin/runc
+# nixpkgs: we build runc separately
echo Building containerd-shim
rm -f ./vendor/github.com/containerd/containerd/bin/containerd-shim
diff --git a/scripts/package-cli b/scripts/package-cli
index ab4a6dac63..044b5587d0 100755
--- a/scripts/package-cli
+++ b/scripts/package-cli
@@ -50,15 +50,17 @@ fi
CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}
-"${GO}" generate
+CGO_ENABLED=0 env -u GOARCH "${GO}" generate
LDFLAGS="
-X github.com/rancher/k3s/pkg/version.Version=$VERSION
-X github.com/rancher/k3s/pkg/version.GitCommit=${COMMIT:0:8}
-w -s
"
-STATIC="-extldflags '-static'"
-CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
+# STATIC="-extldflags '-static'"
+# nixpkgs: we can depend on dynamic linking because we have a good package manager
+"${GO}" build -ldflags "$LDFLAGS" -o ${CMD_NAME} ./cmd/k3s/main.go
stat ${CMD_NAME}
-./scripts/build-upload ${CMD_NAME} ${COMMIT}
+# nixpkgs: skip uploading
+# ./scripts/build-upload ${CMD_NAME} ${COMMIT}

View file

@ -0,0 +1,37 @@
From 49c000c7c5dd7a502a2be4c638d2c32b65673c00 Mon Sep 17 00:00:00 2001
From: Euan Kemp <euank@euank.com>
Date: Sun, 6 Feb 2022 23:13:00 -0800
Subject: [PATCH] Don't build a static binary in package-cli
since nixpkgs prefers dynamically linked binaries.
Also remove "trimpath" for the 'go generate' step because the codegen
they use doesn't work with trimpath set.
---
scripts/package-cli | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/scripts/package-cli b/scripts/package-cli
index 28927327b7..95dbb469f1 100755
--- a/scripts/package-cli
+++ b/scripts/package-cli
@@ -48,14 +48,13 @@ fi
CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}
-"${GO}" generate
+GOFLAGS="" "${GO}" generate
LDFLAGS="
-X github.com/rancher/k3s/pkg/version.Version=$VERSION
-X github.com/rancher/k3s/pkg/version.GitCommit=${COMMIT:0:8}
-w -s
"
-STATIC="-extldflags '-static'"
-CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
+CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS" -o ${CMD_NAME} ./cmd/k3s/main.go
stat ${CMD_NAME}
--
2.34.1

View file

@ -26855,7 +26855,7 @@ with pkgs;
jwm-settings-manager = callPackage ../applications/window-managers/jwm/jwm-settings-manager.nix { };
k3s = callPackage ../applications/networking/cluster/k3s {};
k3s = callPackage ../applications/networking/cluster/k3s { };
kconf = callPackage ../applications/networking/cluster/kconf { };