minimal-bootstrap.mes: generate list of source files

pkgs/os-specific/linux/minimal-bootstrap/default.nix

@@ -17,7 +17,7 @@ lib.makeScope
     inherit (callPackage ./stage0-posix { }) kaem m2libc mescc-tools mescc-tools-extra writeTextFile writeText runCommand;
 
     mes = callPackage ./mes { };
-    inherit (mes) mes-libc;
+    mes-libc = callPackage ./mes/libc.nix { };
 
     ln-boot = callPackage ./ln-boot { };
 

pkgs/os-specific/linux/minimal-bootstrap/mes/default.nix

@@ -22,106 +22,11 @@ let
     #define MES_VERSION "${version}"
   '';
 
-  # Maintenance note:
-  # Build steps have been adapted from build-aux/bootstrap.sh.in
-  # as well as the live-bootstrap project
-  # https://github.com/fosslinux/live-bootstrap/blob/1bc4296091c51f53a5598050c8956d16e945b0f5/sysa/mes-0.24.2/mes-0.24.2.kaem
+  sources = (import ./sources.nix).x86.linux.mescc;
+  inherit (sources) libc_mini_SOURCES libmescc_SOURCES libc_SOURCES mes_SOURCES;
 
-  # Maintenance note: list of source files derived from build-aux/configure-lib.sh
-  libc_mini_shared_SOURCES = cc: [
-    "lib/mes/eputs.c"
-    "lib/mes/oputs.c"
-    "lib/mes/globals.c"
-    "lib/stdlib/exit.c"
-    "lib/linux/x86-mes-${cc}/_exit.c"
-    "lib/linux/x86-mes-${cc}/_write.c"
-    "lib/stdlib/puts.c"
-    "lib/string/strlen.c"
-  ];
-  libc_mini_SOURCES = cc: libc_mini_shared_SOURCES cc ++ [
-    "lib/mes/mini-write.c"
-  ];
-  libmescc_SOURCES = cc: [
-    "lib/mes/globals.c"
-    "lib/linux/x86-mes-${cc}/syscall-internal.c"
-  ];
-  libmes_SOURCES = cc: libc_mini_shared_SOURCES cc ++ lib.splitString " " (
-    "lib/ctype/isnumber.c lib/mes/abtol.c lib/mes/cast.c lib/mes/eputc.c lib/mes/fdgetc.c "
-    + "lib/mes/fdputc.c lib/mes/fdputs.c lib/mes/fdungetc.c lib/mes/itoa.c lib/mes/ltoa.c "
-    + "lib/mes/ltoab.c lib/mes/mes_open.c lib/mes/ntoab.c lib/mes/oputc.c lib/mes/ultoa.c "
-    + "lib/mes/utoa.c lib/stub/__raise.c lib/ctype/isdigit.c lib/ctype/isspace.c "
-    + "lib/ctype/isxdigit.c lib/mes/assert_msg.c lib/posix/write.c lib/stdlib/atoi.c "
-    + "lib/linux/lseek.c");
-  libc_SOURCES = cc: libmes_SOURCES cc ++ lib.splitString " " (
-    "lib/mes/__assert_fail.c lib/mes/__buffered_read.c lib/mes/__mes_debug.c "
-    + "lib/posix/execv.c lib/posix/getcwd.c lib/posix/getenv.c lib/posix/isatty.c "
-    + "lib/posix/open.c lib/posix/buffered-read.c lib/posix/setenv.c lib/posix/wait.c "
-    + "lib/stdio/fgetc.c lib/stdio/fputc.c lib/stdio/fputs.c lib/stdio/getc.c "
-    + "lib/stdio/getchar.c lib/stdio/putc.c lib/stdio/putchar.c lib/stdio/ungetc.c "
-    + "lib/stdlib/free.c lib/stdlib/realloc.c lib/string/memchr.c lib/string/memcmp.c "
-    + "lib/string/memcpy.c lib/string/memmove.c lib/string/memset.c lib/string/strcmp.c "
-    + "lib/string/strcpy.c lib/string/strncmp.c lib/posix/raise.c "
-    + "lib/linux/access.c lib/linux/brk.c lib/linux/chmod.c lib/linux/clock_gettime.c "
-    + "lib/linux/dup.c lib/linux/dup2.c lib/linux/execve.c lib/linux/fork.c lib/linux/fsync.c "
-    + "lib/linux/_getcwd.c lib/linux/gettimeofday.c lib/linux/ioctl3.c lib/linux/_open3.c "
-    + "lib/linux/malloc.c lib/linux/_read.c lib/linux/time.c lib/linux/unlink.c "
-    + "lib/linux/waitpid.c lib/linux/x86-mes-${cc}/syscall.c lib/linux/getpid.c "
-    + "lib/linux/kill.c");
-  libc_tcc_SOURCES = cc: lib.splitString " " (
-      "lib/ctype/islower.c lib/ctype/isupper.c lib/ctype/tolower.c lib/ctype/toupper.c "
-      + "lib/mes/abtod.c lib/mes/dtoab.c lib/mes/search-path.c lib/posix/execvp.c "
-      + "lib/stdio/fclose.c lib/stdio/fdopen.c lib/stdio/ferror.c lib/stdio/fflush.c "
-      + "lib/stdio/fopen.c lib/stdio/fprintf.c lib/stdio/fread.c lib/stdio/fseek.c "
-      + "lib/stdio/ftell.c lib/stdio/fwrite.c lib/stdio/printf.c lib/stdio/remove.c "
-      + "lib/stdio/snprintf.c lib/stdio/sprintf.c lib/stdio/sscanf.c lib/stdio/vfprintf.c "
-      + "lib/stdio/vprintf.c lib/stdio/vsnprintf.c lib/stdio/vsprintf.c lib/stdio/vsscanf.c "
-      + "lib/stdlib/calloc.c lib/stdlib/qsort.c lib/stdlib/strtod.c lib/stdlib/strtof.c "
-      + "lib/stdlib/strtol.c lib/stdlib/strtold.c lib/stdlib/strtoll.c lib/stdlib/strtoul.c "
-      + "lib/stdlib/strtoull.c lib/string/memmem.c lib/string/strcat.c lib/string/strchr.c "
-      + "lib/string/strlwr.c lib/string/strncpy.c lib/string/strrchr.c lib/string/strstr.c "
-      + "lib/string/strupr.c lib/stub/sigaction.c lib/stub/ldexp.c lib/stub/mprotect.c "
-      + "lib/stub/localtime.c lib/stub/sigemptyset.c lib/x86-mes-${cc}/setjmp.c "
-      + "lib/linux/close.c lib/linux/rmdir.c lib/linux/stat.c"
-    ) ++ [
-      # add symlink() to libc+tcc so we can use it in ln-boot
-      "lib/linux/symlink.c"
-    ];
-  libc_gnu_SOURCES = cc: libc_tcc_SOURCES cc ++ lib.splitString " " (
-    "lib/ctype/isalnum.c lib/ctype/isalpha.c lib/ctype/isascii.c lib/ctype/iscntrl.c "
-    + "lib/ctype/isgraph.c lib/ctype/isprint.c lib/ctype/ispunct.c lib/dirent/__getdirentries.c "
-    + "lib/dirent/closedir.c lib/dirent/opendir.c lib/dirent/readdir.c lib/math/ceil.c "
-    + "lib/math/fabs.c lib/math/floor.c lib/mes/fdgets.c lib/posix/alarm.c lib/posix/execl.c "
-    + "lib/posix/execlp.c lib/posix/mktemp.c lib/posix/sbrk.c lib/posix/sleep.c "
-    + "lib/posix/unsetenv.c lib/stdio/clearerr.c lib/stdio/feof.c lib/stdio/fgets.c "
-    + "lib/stdio/fileno.c lib/stdio/freopen.c lib/stdio/fscanf.c lib/stdio/perror.c "
-    + "lib/stdio/vfscanf.c lib/stdlib/__exit.c lib/stdlib/abort.c lib/stdlib/abs.c "
-    + "lib/stdlib/alloca.c lib/stdlib/atexit.c lib/stdlib/atof.c lib/stdlib/atol.c "
-    + "lib/stdlib/mbstowcs.c lib/string/bcmp.c lib/string/bcopy.c lib/string/bzero.c "
-    + "lib/string/index.c lib/string/rindex.c lib/string/strcspn.c lib/string/strdup.c "
-    + "lib/string/strerror.c lib/string/strncat.c lib/string/strpbrk.c lib/string/strspn.c "
-    + "lib/stub/__cleanup.c lib/stub/atan2.c lib/stub/bsearch.c lib/stub/chown.c "
-    + "lib/stub/cos.c lib/stub/ctime.c lib/stub/exp.c lib/stub/fpurge.c lib/stub/freadahead.c "
-    + "lib/stub/frexp.c lib/stub/getgrgid.c lib/stub/getgrnam.c lib/stub/getlogin.c "
-    + "lib/stub/getpgid.c lib/stub/getpgrp.c lib/stub/getpwnam.c lib/stub/getpwuid.c "
-    + "lib/stub/gmtime.c lib/stub/log.c lib/stub/mktime.c lib/stub/modf.c lib/stub/pclose.c "
-    + "lib/stub/popen.c lib/stub/pow.c lib/stub/rand.c lib/stub/rewind.c lib/stub/setbuf.c "
-    + "lib/stub/setgrent.c lib/stub/setlocale.c lib/stub/setvbuf.c lib/stub/sigaddset.c "
-    + "lib/stub/sigblock.c lib/stub/sigdelset.c lib/stub/sigsetmask.c lib/stub/sin.c "
-    + "lib/stub/sqrt.c lib/stub/strftime.c lib/stub/sys_siglist.c lib/stub/system.c "
-    + "lib/stub/times.c lib/stub/ttyname.c lib/stub/umask.c lib/stub/utime.c "
-    + "lib/linux/chdir.c lib/linux/fcntl.c lib/linux/fstat.c lib/linux/getdents.c "
-    + "lib/linux/getegid.c lib/linux/geteuid.c lib/linux/getgid.c lib/linux/getppid.c "
-    + "lib/linux/getrusage.c lib/linux/getuid.c lib/linux/ioctl.c lib/linux/link.c "
-    + "lib/linux/lstat.c lib/linux/mkdir.c lib/linux/mknod.c lib/linux/nanosleep.c "
-    + "lib/linux/pipe.c lib/linux/readlink.c lib/linux/rename.c lib/linux/setgid.c "
-    + "lib/linux/settimer.c lib/linux/setuid.c lib/linux/signal.c lib/linux/sigprogmask.c "
-    # sylink.c already included above in libc_tcc_SOURCES
-    # + "lib/linux/symlink.c"
-  );
-  mes_SOURCES = cc: lib.splitString " " (
-    "src/builtins.c src/cc.c src/core.c src/display.c src/eval-apply.c src/gc.c "
-    + "src/globals.c src/hash.c src/lib.c src/math.c src/mes.c src/module.c src/posix.c "
-    + "src/reader.c src/stack.c src/string.c src/struct.c src/symbol.c src/vector.c");
+  # add symlink() to libc+tcc so we can use it in ln-boot
+  libc_tcc_SOURCES = sources.libc_tcc_SOURCES ++ [ "lib/linux/symlink.c" ];
 
   compile = sources:
     lib.concatMapStringsSep
@@ -141,9 +46,7 @@ in
 runCommand "${pname}-${version}" {
   inherit pname version;
 
-  passthru.mes-libc = callPackage ./libc.nix {
-    inherit libc_SOURCES libc_gnu_SOURCES;
-  };
+  passthru = { inherit src nyacc; };
 
   meta = with lib; {
     description = "Scheme interpreter and C compiler for bootstrapping";
@@ -152,7 +55,12 @@ runCommand "${pname}-${version}" {
     maintainers = with maintainers; [ emilytrau ];
     platforms = [ "i686-linux" ];
   };
-} ''
+}
+# Maintenance note:
+# Build steps have been adapted from build-aux/bootstrap.sh.in
+# as well as the live-bootstrap project
+# https://github.com/fosslinux/live-bootstrap/blob/1bc4296091c51f53a5598050c8956d16e945b0f5/sysa/mes-0.24.2/mes-0.24.2.kaem
+''
   # Unpack source
   ungz --file ${src} --output mes.tar
   mkdir ''${out} ''${out}/bin ''${out}/share
@@ -235,35 +143,36 @@ runCommand "${pname}-${version}" {
   cp crt1.s ''${LIBDIR}/x86-mes
 
   # libc-mini.a
-  ${compile (libc_mini_SOURCES "mescc")}
-  ${archive "libc-mini.a" (libc_mini_SOURCES "mescc")}
-  ${sourceArchive "libc-mini.s" (libc_mini_SOURCES "mescc")}
+  ${compile libc_mini_SOURCES}
+  ${archive "libc-mini.a" libc_mini_SOURCES}
+  ${sourceArchive "libc-mini.s" libc_mini_SOURCES}
   cp libc-mini.a ''${LIBDIR}/x86-mes
   cp libc-mini.s ''${LIBDIR}/x86-mes
 
   # libmescc.a
-  ${compile (libmescc_SOURCES "mescc")}
-  ${archive "libmescc.a" (libmescc_SOURCES "mescc")}
-  ${sourceArchive "libmescc.s" (libmescc_SOURCES "mescc")}
+  ${compile libmescc_SOURCES}
+  ${archive "libmescc.a" libmescc_SOURCES}
+  ${sourceArchive "libmescc.s" libmescc_SOURCES}
   cp libmescc.a ''${LIBDIR}/x86-mes
   cp libmescc.s ''${LIBDIR}/x86-mes
 
   # libc.a
-  ${compile (libc_SOURCES "mescc")}
-  ${archive "libc.a" (libc_SOURCES "mescc")}
-  ${sourceArchive "libc.s" (libc_SOURCES "mescc")}
+  ${compile libc_SOURCES}
+  ${archive "libc.a" libc_SOURCES}
+  ${sourceArchive "libc.s" libc_SOURCES}
   cp libc.a ''${LIBDIR}/x86-mes
   cp libc.s ''${LIBDIR}/x86-mes
 
   # libc+tcc.a
-  ${compile (libc_tcc_SOURCES "mescc")}
-  ${archive "libc+tcc.a" ([ "libc.a" ] ++ libc_tcc_SOURCES "mescc")}
-  ${sourceArchive "libc+tcc.s" ([ "libc.s" ] ++ libc_tcc_SOURCES "mescc")}
+  # optimisation: don't recompile common libc sources
+  ${compile (lib.subtractLists libc_SOURCES libc_tcc_SOURCES)}
+  ${archive "libc+tcc.a" libc_tcc_SOURCES}
+  ${sourceArchive "libc+tcc.s" libc_tcc_SOURCES}
   cp libc+tcc.a ''${LIBDIR}/x86-mes
   cp libc+tcc.s ''${LIBDIR}/x86-mes
 
   # Build mes itself
-  ${compile (mes_SOURCES "mescc")}
+  ${compile mes_SOURCES}
   ''${out}/bin/mes-m2 -e main ''${out}/bin/mescc.scm -- \
     --base-address 0x08048000 \
     -L ''${MES_PREFIX}/lib \
@@ -273,7 +182,7 @@ runCommand "${pname}-${version}" {
     -nostdlib \
     -o ''${out}/bin/mes \
     crt1.o \
-    ${lib.concatMapStringsSep " " (replaceExt ".o") (mes_SOURCES "mescc")}
+    ${lib.concatMapStringsSep " " (replaceExt ".o") mes_SOURCES}
 
   # Check
   ''${out}/bin/mes --version

pkgs/os-specific/linux/minimal-bootstrap/mes/gen-sources.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p bash coreutils gnutar
+
+# Generate a sources.nix for a version of GNU mes. Creates lists of source files
+# from build-aux/configure-lib.sh.
+#
+# You may point this tool at a manually downloaded tarball, but more ideal is
+# using the source tarball from Nixpkgs. For example:
+#
+# MES_TARBALL="$(nix-build --no-link -A minimal-bootstrap.mes.src ../../../../..)"
+# ./gen-sources.sh "$MES_TARBALL" > ./new-sources.nix
+
+set -eu
+
+# Supported platforms
+ARCHS="x86"
+KERNELS="linux"
+COMPILERS="mescc gcc"
+
+
+format() {
+  echo -n "[ "
+  # Terrible hack to convert a newline-delimited string to space-delimited
+  echo $* | xargs printf '"%s" '
+  echo -n "]"
+}
+
+gen_sources() {
+  # Configuration variables used by configure-lib.sh
+  export mes_libc=mes
+  export mes_cpu=$1
+  export mes_kernel=$2
+  export compiler=$3
+
+  # Populate source file lists
+  source $CONFIGURE_LIB_SH
+
+  cat <<EOF
+  $mes_cpu.$mes_kernel.$compiler = {
+    libc_mini_SOURCES = $(format $libc_mini_SOURCES);
+    libmescc_SOURCES  = $(format $libmescc_SOURCES);
+    libtcc1_SOURCES   = $(format $libtcc1_SOURCES);
+    libc_SOURCES      = $(format $libc_SOURCES);
+    libc_tcc_SOURCES  = $(format $libc_tcc_SOURCES);
+    libc_gnu_SOURCES  = $(format $libc_gnu_SOURCES);
+    mes_SOURCES       = $(format $mes_SOURCES);
+  };
+EOF
+}
+
+
+MES_TARBALL=$1
+if [ ! -f $MES_TARBALL ]; then
+    echo "Provide path to mes-x.x.x.tar.gz as first argument" >&2
+    exit 1
+fi
+echo "Generating sources.nix from $MES_TARBALL" >&2
+
+TMP=$(mktemp -d)
+cd $TMP
+echo "Workdir: $TMP" >&2
+
+echo "Extracting $MES_TARBALL" >&2
+tar --strip-components 1 -xf $MES_TARBALL
+
+CONFIGURE_LIB_SH="$TMP/build-aux/configure-lib.sh"
+if [ ! -f $CONFIGURE_LIB_SH ]; then
+    echo "Could not find mes's configure-lib.sh script at $CONFIGURE_LIB_SH" >&2
+    exit 1
+fi
+
+# Create dummy config expected by configure-lib.sh
+touch config.sh
+chmod +x config.sh
+
+
+echo "Configuring with $CONFIGURE_LIB_SH" >&2
+
+cat <<EOF
+# This file is generated by ./gen-sources.sh.
+# Do not edit!
+{
+EOF
+
+for arch in $ARCHS; do
+  for kernel in $KERNELS; do
+    for compiler in $COMPILERS; do
+      gen_sources $arch $kernel $compiler
+    done
+  done
+done
+
+cat <<EOF
+}
+EOF

pkgs/os-specific/linux/minimal-bootstrap/mes/libc.nix

@@ -2,21 +2,25 @@
 , runCommand
 , ln-boot
 , mes
-, libc_SOURCES
-, libc_gnu_SOURCES
 , mes-libc
 }:
 let
   pname = "mes-libc";
   inherit (mes) version;
 
+  sources = (import ./sources.nix).x86.linux.gcc;
+  inherit (sources) libtcc1_SOURCES libc_gnu_SOURCES;
+
+  prefix = "${mes}/share/mes-${version}";
+
   # Concatenate all source files into a convenient bundle
-  # "gcc" variants (eg. "lib/linux/x86-mes-gcc") can also be used with tinycc
-  sources = libc_SOURCES "gcc" ++ libc_gnu_SOURCES "gcc";
+  # "gcc" variants of source files (eg. "lib/linux/x86-mes-gcc") can also be
+  # compiled by tinycc
+  #
   # Passing this many arguments is too much for kaem so we need to split
   # the operation in two
-  firstLibc = lib.take 100 sources;
-  lastLibc = lib.drop 100 sources;
+  firstLibc = lib.take 100 libc_gnu_SOURCES;
+  lastLibc = lib.drop 100 libc_gnu_SOURCES;
 in runCommand "${pname}-${version}" {
   inherit pname version;
 
@@ -32,12 +36,10 @@ in runCommand "${pname}-${version}" {
     platforms = [ "i686-linux" ];
   };
 } ''
-  mkdir -p ''${out}/lib
-  PREFIX=${mes}/share/mes-${version}
-  cd ''${PREFIX}
+  cd ${prefix}
 
   # mescc compiled libc.a
-  mkdir ''${out}/lib/x86-mes
+  mkdir -p ''${out}/lib/x86-mes
   cp lib/x86-mes/libc.a ''${out}/lib/x86-mes
 
   # libc.c
@@ -50,11 +52,11 @@ in runCommand "${pname}-${version}" {
   cp lib/linux/x86-mes-gcc/crti.c ''${out}/lib
 
   # libtcc1.c
-  cp lib/libtcc1.c ''${out}/lib
+  catm ''${out}/lib/libtcc1.c ${lib.concatStringsSep " " libtcc1_SOURCES}
 
   # getopt.c
-  cp lib/posix/getopt.c ''${out}/lib
+  cp lib/posix/getopt.c ''${out}/lib/libgetopt.c
 
   # Install headers
-  ln -s ''${PREFIX}/include ''${out}/include
+  ln -s ${prefix}/include ''${out}/include
 ''

pkgs/os-specific/linux/minimal-bootstrap/tinycc/bootstrappable.nix

@@ -80,8 +80,8 @@ let
     ''${out}/bin/tcc -ar cr ''${out}/lib/libc.a libc.o
     ''${out}/bin/tcc ${mes-libc.CFLAGS} -c -o libtcc1.o ${mes-libc}/lib/libtcc1.c
     ''${out}/bin/tcc -ar cr ''${out}/lib/libtcc1.a libtcc1.o
-    ''${out}/bin/tcc ${mes-libc.CFLAGS} -c -o getopt.o ${mes-libc}/lib/getopt.c
-    ''${out}/bin/tcc -ar cr ''${out}/lib/libgetopt.a getopt.o
+    ''${out}/bin/tcc ${mes-libc.CFLAGS} -c -o libgetopt.o ${mes-libc}/lib/libgetopt.c
+    ''${out}/bin/tcc -ar cr ''${out}/lib/libgetopt.a libgetopt.o
   '';
 
   # Bootstrap stage build flags obtained from

pkgs/os-specific/linux/minimal-bootstrap/tinycc/common.nix

@@ -60,8 +60,8 @@
       ''${out}/bin/tcc -ar cr ''${out}/lib/libtcc1.a libtcc1.o
       ''${out}/bin/tcc ${mes-libc.CFLAGS} -c -o libc.o ${mes-libc}/lib/libc.c
       ''${out}/bin/tcc -ar cr ''${out}/lib/libc.a libc.o
-      ''${out}/bin/tcc ${mes-libc.CFLAGS} -c -o getopt.o ${mes-libc}/lib/getopt.c
-      ''${out}/bin/tcc -ar cr ''${out}/lib/libgetopt.a getopt.o
+      ''${out}/bin/tcc ${mes-libc.CFLAGS} -c -o libgetopt.o ${mes-libc}/lib/libgetopt.c
+      ''${out}/bin/tcc -ar cr ''${out}/lib/libgetopt.a libgetopt.o
 
       # Install headers
       ln -s ${mes-libc}/include ''${out}/include