b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

apptainer, singularity: fix defaultPath and reflect upstream changes

Browse source 
Upstream changes:
singularity 3.8.7 (the legacy) -> apptainer 1.1.3 (the renamed) / singularity 3.10.4 (Sylabs's fork)

Build process:
*   Share between different sources
*   Fix the sed regexp to make defaultPath patch work
*   allowGoReference is now true
*   Provied input parameter removeCompat (default to false)
    that removes the compatible "*singularity*" symbolic links
    and related autocompletion files when projectName != "singularity"
*   Change localstatedir to /var/lib
*   Format with nixpkgs-fmt
*   Fix the defaultPath patching
    and use it instead of the `<executable> path` config directive
    deprecated in Apptainer
*   Provide dependencies for new functionalities such as
    squashfuse (unprivileged squashfs mount)
*   Provide an attribute `defaultPathInputs` to override
    prefix of container runtime default PATH

NixOS module programs.singularity:
*   Allow users to specify packages
*   Place related directories to /var/lib
*   Format with nixpkgs-fmt

singularity-tools:
*   Allow users to specify packages
*   Place related directories to /var/lib when building images in VM
This commit is contained in:
Yueh-Shun Li 2023-01-29 01:02:47 +08:00
parent ef09cfec0b
commit 50788d2fb0
8 changed files with 423 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
nixos/doc/manual/from_md/release-notes/rl-2305.section.xml
View file

@ -922,6 +922,42 @@
as general purpose ephemeral block devices has been removed.
</para>
</listitem>
<listitem>
<para>
As Singularity has renamed to
<link xlink:href="https://apptainer.org/news/community-announcement-20211130">Apptainer</link>
to distinguish from
<link xlink:href="https://sylabs.io/2021/05/singularity-community-edition">an
un-renamed fork by Sylabs Inc.</link>, there are now two
packages of Singularity/Apptainer:
</para>
<itemizedlist spacing="compact">
<listitem>
<para>
<literal>apptainer</literal>: From
<literal>github.com/apptainer/apptainer</literal>, which
is the new repo after renaming.
</para>
</listitem>
<listitem>
<para>
<literal>singularity</literal>: From
<literal>github.com/sylabs/singularity</literal>, which is
the fork by Sylabs Inc..
</para>
</listitem>
</itemizedlist>
<para>
<literal>programs.singularity</literal> got a new
<literal>package</literal> option to specify which package to
use.
</para>
<para>
<literal>singularity-tools.buildImage</literal> got a new
input argument <literal>singularity</literal> to specify which
package to use.
</para>
</listitem>
<listitem>
<para>
The <literal>unifi-poller</literal> package and corresponding

10
nixos/doc/manual/release-notes/rl-2305.section.md
View file

@ -225,6 +225,16 @@ In addition to numerous new and upgraded packages, this release has the followin
- The `zramSwap` is now implemented with `zram-generator`, and the option `zramSwap.numDevices` for using ZRAM devices as general purpose ephemeral block devices has been removed.
- As Singularity has renamed to [Apptainer](https://apptainer.org/news/community-announcement-20211130)
to distinguish from [an un-renamed fork by Sylabs Inc.](https://sylabs.io/2021/05/singularity-community-edition),
there are now two packages of Singularity/Apptainer:
* `apptainer`: From `github.com/apptainer/apptainer`, which is the new repo after renaming.
* `singularity`: From `github.com/sylabs/singularity`, which is the fork by Sylabs Inc..
`programs.singularity` got a new `package` option to specify which package to use.
`singularity-tools.buildImage` got a new input argument `singularity` to specify which package to use.
- The `unifi-poller` package and corresponding NixOS module have been renamed to `unpoller` to match upstream.
- The new option `services.tailscale.useRoutingFeatures` controls various settings for using Tailscale features like exit nodes and subnet routers. If you wish to use your machine as an exit node, you can set this setting to `server`, otherwise if you wish to use an exit node you can set this setting to `client`. The strict RPF warning has been removed as the RPF will be loosened automatically based on the value of this setting.

84
nixos/modules/programs/singularity.nix
View file

@ -3,34 +3,78 @@
with lib;
let
cfg = config.programs.singularity;
singularity = pkgs.singularity.overrideAttrs (attrs: {
installPhase = attrs.installPhase + ''
mv $out/libexec/singularity/bin/starter-suid $out/libexec/singularity/bin/starter-suid.orig
ln -s /run/wrappers/bin/singularity-suid $out/libexec/singularity/bin/starter-suid
'';
});
in
{
options.programs.singularity = {
enable = mkEnableOption (lib.mdDoc "Singularity");
enable = mkEnableOption (mdDoc "singularity") // {
description = mdDoc ''
Whether to install Singularity/Apptainer with system-level overriding such as SUID support.
'';
};
package = mkOption {
type = types.package;
default = pkgs.singularity;
defaultText = literalExpression "pkgs.singularity";
example = literalExpression "pkgs.apptainer";
description = mdDoc ''
Singularity/Apptainer package to override and install.
'';
};
packageOverriden = mkOption {
type = types.nullOr types.package;
default = null;
description = mdDoc ''
This option provides access to the overriden result of `programs.singularity.package`.
For example, the following configuration makes all the Nixpkgs packages use the overriden `singularity`:
```Nix
{ config, lib, pkgs, ... }:
{
nixpkgs.overlays = [
(final: prev: {
_singularity-orig = prev.singularity;
singularity = config.programs.singularity.packageOverriden;
})
];
programs.singularity.enable = true;
programs.singularity.package = pkgs._singularity-orig;
}
```
Use `lib.mkForce` to forcefully specify the overriden package.
'';
};
enableSuid = mkOption {
type = types.bool;
default = true;
example = false;
description = mdDoc ''
Whether to enable the SUID support of Singularity/Apptainer.
'';
};
};
config = mkIf cfg.enable {
environment.systemPackages = [ singularity ];
security.wrappers.singularity-suid =
{
setuid = true;
owner = "root";
group = "root";
source = "${singularity}/libexec/singularity/bin/starter-suid.orig";
};
programs.singularity.packageOverriden = (cfg.package.override (
optionalAttrs cfg.enableSuid {
enableSuid = true;
starterSuidPath = "/run/wrappers/bin/${cfg.package.projectName}-suid";
}
));
environment.systemPackages = [ cfg.packageOverriden ];
security.wrappers."${cfg.packageOverriden.projectName}-suid" = mkIf cfg.enableSuid {
setuid = true;
owner = "root";
group = "root";
source = "${cfg.packageOverriden}/libexec/${cfg.packageOverriden.projectName}/bin/starter-suid.orig";
};
systemd.tmpfiles.rules = [
"d /var/singularity/mnt/session 0770 root root -"
"d /var/singularity/mnt/final 0770 root root -"
"d /var/singularity/mnt/overlay 0770 root root -"
"d /var/singularity/mnt/container 0770 root root -"
"d /var/singularity/mnt/source 0770 root root -"
"d /var/lib/${cfg.packageOverriden.projectName}/mnt/session 0770 root root -"
"d /var/lib/${cfg.packageOverriden.projectName}/mnt/final 0770 root root -"
"d /var/lib/${cfg.packageOverriden.projectName}/mnt/overlay 0770 root root -"
"d /var/lib/${cfg.packageOverriden.projectName}/mnt/container 0770 root root -"
"d /var/lib/${cfg.packageOverriden.projectName}/mnt/source 0770 root root -"
];
};

73
pkgs/applications/virtualization/singularity/default.nix
View file

@ -1,73 +0,0 @@
{ lib
, fetchurl
, util-linux
, gpgme
, openssl
, libuuid
, coreutils
, which
, makeWrapper
, cryptsetup
, squashfsTools
, buildGoPackage}:
with lib;
buildGoPackage rec {
pname = "singularity";
version = "3.8.7";
src = fetchurl {
url = "https://github.com/hpcng/singularity/releases/download/v${version}/singularity-${version}.tar.gz";
sha256 = "sha256-Myny5YP4SoNDyywDgKHWy86vrn0eYztcvK33FD6shZs=";
};
goPackagePath = "github.com/sylabs/singularity";
buildInputs = [ gpgme openssl libuuid ];
nativeBuildInputs = [ util-linux which makeWrapper cryptsetup ];
propagatedBuildInputs = [ coreutils squashfsTools ];
postPatch = ''
substituteInPlace internal/pkg/build/files/copy.go \
--replace /bin/cp ${coreutils}/bin/cp
'';
postConfigure = ''
cd go/src/github.com/sylabs/singularity
patchShebangs .
sed -i 's|defaultPath := "[^"]*"|defaultPath := "${lib.makeBinPath propagatedBuildInputs}"|' cmd/internal/cli/actions.go
./mconfig -V ${version} -p $out --localstatedir=/var
# Don't install SUID binaries
sed -i 's/-m 4755/-m 755/g' builddir/Makefile
'';
buildPhase = ''
runHook preBuild
make -C builddir
runHook postBuild
'';
installPhase = ''
runHook preInstall
make -C builddir install LOCALSTATEDIR=$out/var
chmod 755 $out/libexec/singularity/bin/starter-suid
# Explicitly configure paths in the config file
sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${lib.makeBinPath [squashfsTools]}/mksquashfs|' $out/etc/singularity/singularity.conf
sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [cryptsetup]}/cryptsetup|' $out/etc/singularity/singularity.conf
runHook postInstall
'';
meta = with lib; {
homepage = "http://www.sylabs.io/";
description = "Application containers for linux";
license = licenses.bsd3;
platforms = platforms.linux;
maintainers = [ maintainers.jbedo ];
};
}

222
pkgs/applications/virtualization/singularity/generic.nix Normal file
View file

@ -0,0 +1,222 @@
# Configurations that should only be overrided by
# overrideAttrs
{ pname
, version
, src
, projectName # "apptainer" or "singularity"
, vendorHash ? null
, deleteVendor ? false
, proxyVendor ? false
, extraConfigureFlags ? [ ]
, extraDescription ? ""
, extraMeta ? { }
}:
let
# Workaround for vendor-related attributes not overridable (#86349)
# should be removed when the issue is resolved
_defaultGoVendorArgs = {
inherit
vendorHash
deleteVendor
proxyVendor
;
};
in
{ lib
, buildGoModule
# Native build inputs
, makeWrapper
, pkg-config
, util-linux
, which
# Build inputs
, bash
, conmon
, coreutils
, cryptsetup
, fakeroot
, go
, gpgme
, libseccomp
, libuuid
# This is for nvidia-container-cli
, nvidia-docker
, openssl
, squashfsTools
, squashfuse
# Overridable configurations
, enableNvidiaContainerCli ? true
# Compile with seccomp support
# SingularityCE 3.10.0 and above requires explicit --without-seccomp when libseccomp is not available.
, enableSeccomp ? true
# Whether the configure script treat SUID support as default
, defaultToSuid ? true
# Whether to compile with SUID support
, enableSuid ? false
, starterSuidPath ? null
# Remove the symlinks to `singularity*` when projectName != "singularity"
, removeCompat ? false
# Workaround #86349
# should be removed when the issue is resolved
, vendorHash ? _defaultGoVendorArgs.vendorHash
, deleteVendor ? _defaultGoVendorArgs.deleteVendor
, proxyVendor ? _defaultGoVendorArgs.proxyVendor
}:
let
defaultPathOriginal = "/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin";
in
buildGoModule {
inherit pname version src;
# Override vendorHash with the output got from
# nix-prefetch -E "{ sha256 }: ((import ./. { }).apptainer.override { vendorHash = sha256; }).go-modules"
# or with `null` when using vendored source tarball.
inherit vendorHash deleteVendor proxyVendor;
# go is used to compile extensions when building container images
allowGoReference = true;
strictDeps = true;
passthru = {
inherit
enableSeccomp
enableSuid
projectName
removeCompat
starterSuidPath
;
};
nativeBuildInputs = [
makeWrapper
pkg-config
util-linux
which
];
buildInputs = [
bash # To patch /bin/sh shebangs.
conmon
cryptsetup
gpgme
libuuid
openssl
squashfsTools
squashfuse
]
++ lib.optional enableNvidiaContainerCli nvidia-docker
++ lib.optional enableSeccomp libseccomp
;
configureScript = "./mconfig";
configureFlags = [
"--localstatedir=/var/lib"
"--runstatedir=/var/run"
]
++ lib.optional (!enableSeccomp) "--without-seccomp"
++ lib.optional (defaultToSuid && !enableSuid) "--without-suid"
++ lib.optional (!defaultToSuid && enableSuid) "--with-suid"
++ extraConfigureFlags
;
# Packages to prefix to the Apptainer/Singularity container runtime default PATH
# Use overrideAttrs to override
defaultPathInputs = [
bash
coreutils
cryptsetup # cryptsetup
go
squashfsTools # mksquashfs unsquashfs # Make / unpack squashfs image
squashfuse # squashfuse_ll squashfuse # Mount (without unpacking) a squashfs image without privileges
]
++ lib.optional enableNvidiaContainerCli nvidia-docker
;
postPatch = ''
if [[ ! -e .git || ! -e VERSION ]]; then
echo "${version}" > VERSION
fi
# Patch shebangs for script run during build
patchShebangs --build "$configureScript" makeit e2e scripts mlocal/scripts
# Patching the hard-coded defaultPath by prefixing the packages in defaultPathInputs
substituteInPlace cmd/internal/cli/actions.go \
--replace "defaultPath = \"${defaultPathOriginal}\"" "defaultPath = \"''${defaultPathInputs// /\/bin:}''${defaultPathInputs:+/bin:}${defaultPathOriginal}\""
'';
postConfigure = ''
# Code borrowed from pkgs/stdenv/generic/setup.sh configurePhase()
# set to empty if unset
: ''${configureFlags=}
# shellcheck disable=SC2086
$configureScript -V ${version} "''${prefixKey:---prefix=}$prefix" $configureFlags "''${configureFlagsArray[@]}"
# End of the code from pkgs/stdenv/generic/setup.sh configurPhase()
'';
buildPhase = ''
runHook preBuild
make -C builddir -j"$NIX_BUILD_CORES"
runHook postBuild
'';
installPhase = ''
runHook preInstall
make -C builddir install LOCALSTATEDIR="$out/var/lib"
runHook postInstall
'';
postFixup = ''
substituteInPlace "$out/bin/run-singularity" \
--replace "/usr/bin/env ${projectName}" "$out/bin/${projectName}"
wrapProgram "$out/bin/${projectName}" \
--prefix PATH : "${lib.makeBinPath [
fakeroot
squashfsTools # Singularity (but not Apptainer) expects unsquashfs from the host PATH
]}"
# Make changes in the config file
${lib.optionalString enableNvidiaContainerCli ''
substituteInPlace "$out/etc/${projectName}/${projectName}.conf" \
--replace "use nvidia-container-cli = no" "use nvidia-container-cli = yes"
''}
${lib.optionalString (removeCompat && (projectName != "singularity")) ''
unlink "$out/bin/singularity"
for file in "$out"/share/man/man?/singularity*.gz; do
if [[ -L "$file" ]]; then
unlink "$file"
fi
done
for file in "$out"/share/*-completion/completions/singularity; do
if [[ -e "$file" ]]
rm "$file"
done
''}
${lib.optionalString enableSuid (lib.warnIf (isNull starterSuidPath) "${projectName}: Null starterSuidPath when enableSuid produces non-SUID-ed starter-suid and run-time permission denial." ''
chmod +x $out/libexec/${projectName}/bin/starter-suid
'')}
${lib.optionalString (enableSuid && !isNull starterSuidPath) ''
mv "$out"/libexec/${projectName}/bin/starter-suid{,.orig}
ln -s ${lib.escapeShellArg starterSuidPath} "$out/libexec/${projectName}/bin/starter-suid"
''}
'';
meta = with lib; {
description = "Application containers for linux" + extraDescription;
longDescription = ''
Singularity (the upstream) renamed themselves to Apptainer
to distinguish themselves from a fork made by Sylabs Inc.. See
https://sylabs.io/2021/05/singularity-community-edition
https://apptainer.org/news/community-announcement-20211130
'';
license = licenses.bsd3;
platforms = platforms.linux;
maintainers = with maintainers; [ jbedo ShamrockLee ];
mainProgram = projectName;
} // extraMeta;
}

70
pkgs/applications/virtualization/singularity/packages.nix Normal file
View file

@ -0,0 +1,70 @@
{ callPackage
, fetchFromGitHub
, conmon
}:
{
apptainer = callPackage
(import ./generic.nix rec {
pname = "apptainer";
# TODO: Upgrade to 1.1.4 only after https://github.com/apptainer/apptainer/pull/967 get merge
# and https://github.com/apptainer/apptainer/issues/958 get fixed
version = "1.1.3";
projectName = "apptainer";
src = fetchFromGitHub {
owner = "apptainer";
repo = "apptainer";
rev = "v${version}";
hash = "sha256-QFg6RC77OE/a6Qlzn6Zi5I7Iaq/U3/m0eI9yLArzuNc=";
};
# Update by running
# nix-prefetch -E "{ sha256 }: ((import ./. { }).apptainer.override { vendorHash = sha256; }).go-modules"
# at the root directory of the Nixpkgs repository
vendorHash = "sha256-tAnh7A8Lw5KtY7hq+sqHMEUlgXvgeeCKKIfRZFoRtug=";
extraDescription = " (previously known as Singularity)";
extraMeta.homepage = "https://apptainer.org";
})
{
# Apptainer doesn't depend on conmon
conmon = null;
# defaultToSuid becomes false since Apptainer 1.1.0
# https://github.com/apptainer/apptainer/pull/495
# https://github.com/apptainer/apptainer/releases/tag/v1.1.0
defaultToSuid = false;
};
singularity = callPackage
(import ./generic.nix rec {
pname = "singularity-ce";
version = "3.10.4";
projectName = "singularity";
src = fetchFromGitHub {
owner = "sylabs";
repo = "singularity";
rev = "v${version}";
hash = "sha256-bUnQXQVwaVA3Lkw3X9TBWqNBgiPxAVCHnkq0vc+CIsM=";
};
# Update by running
# nix-prefetch -E "{ sha256 }: ((import ./. { }).singularity.override { vendorHash = sha256; }).go-modules"
# at the root directory of the Nixpkgs repository
vendorHash = "sha256-K8helLcOuz3E4LzBE9y3pnZqwdwhO/iMPTN1o22ipVg=";
# Do not build conmon from the Git submodule source,
# Use Nixpkgs provided version
extraConfigureFlags = [
"--without-conmon"
];
extraDescription = " (Sylabs Inc's fork of Singularity, a.k.a. SingularityCE)";
extraMeta.homepage = "https://sylabs.io/";
})
{
defaultToSuid = true;
};
}

29
pkgs/build-support/singularity-tools/default.nix
View file

@ -23,9 +23,10 @@ rec {
mkLayer =
{ name
, contents ? [ ]
,
# May be "apptainer" instead of "singularity"
, projectName ? (singularity.projectName or "singularity")
}:
runCommand "singularity-layer-${name}"
runCommand "${projectName}-layer-${name}"
{
inherit contents;
} ''
@ -36,28 +37,34 @@ rec {
'';
buildImage =
let
defaultSingularity = singularity;
in
{ name
, contents ? [ ]
, diskSize ? 1024
, runScript ? "#!${stdenv.shell}\nexec /bin/sh"
, runAsRoot ? null
, memSize ? 512
, singularity ? defaultSingularity
}:
let
projectName = singularity.projectName or "singularity";
layer = mkLayer {
inherit name;
contents = contents ++ [ bash runScriptFile ];
inherit projectName;
};
runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
runScriptFile = shellScript "run-script.sh" runScript;
result = vmTools.runInLinuxVM (
runCommand "singularity-image-${name}.img"
runCommand "${projectName}-image-${name}.img"
{
buildInputs = [ singularity e2fsprogs util-linux gawk ];
layerClosure = writeReferencesToFile layer;
preVM = vmTools.createEmptyImage {
size = diskSize;
fullName = "singularity-run-disk";
fullName = "${projectName}-run-disk";
};
inherit memSize;
}
@ -96,18 +103,18 @@ rec {
if [ ! -e bin/sh ]; then
ln -s ${runtimeShell} bin/sh
fi
mkdir -p .singularity.d
ln -s ${runScriptFile} .singularity.d/runscript
mkdir -p .${projectName}.d
ln -s ${runScriptFile} .${projectName}.d/runscript
# Fill out .singularity.d
mkdir -p .singularity.d/env
touch .singularity.d/env/94-appsbase.sh
# Fill out .${projectName}.d
mkdir -p .${projectName}.d/env
touch .${projectName}.d/env/94-appsbase.sh
cd ..
mkdir -p /var/singularity/mnt/{container,final,overlay,session,source}
mkdir -p /var/lib/${projectName}/mnt/{container,final,overlay,session,source}
echo "root:x:0:0:System administrator:/root:/bin/sh" > /etc/passwd
echo > /etc/resolv.conf
TMPDIR=$(pwd -P) singularity build $out ./img
TMPDIR=$(pwd -P) ${projectName} build $out ./img
'');
in

4
pkgs/top-level/all-packages.nix
View file

@ -30196,7 +30196,9 @@ with pkgs;
shepherd = nodePackages."@nerdwallet/shepherd";
singularity = callPackage ../applications/virtualization/singularity { };
inherit (callPackage ../applications/virtualization/singularity/packages.nix { })
apptainer
singularity;
skate = callPackage ../applications/misc/skate { };