Merge commit '8e2b14aceb1d40c7e8b84c03a7c78955359872bb'

10 files changed, 160 insertions, 77 deletions

diff --git a/nixpkgs/nixos/lib/make-disk-image.nix b/nixpkgs/nixos/lib/make-disk-image.nix
index 5e86ea479d5..8aa606a56af 100644
--- a/nixpkgs/nixos/lib/make-disk-image.nix
+++ b/nixpkgs/nixos/lib/make-disk-image.nix
@@ -5,21 +5,32 @@
   config
 
 , # The size of the disk, in megabytes.
-  diskSize
+  # if "auto" size is calculated based on the contents copied to it and
+  #   additionalSpace is taken into account.
+  diskSize ? "auto"
 
-  # The files and directories to be placed in the target file system.
+, # additional disk space to be added to the image if diskSize "auto"
+  # is used
+  additionalSpace ? "512M"
+
+, # size of the boot partition, is only used if partitionTableType is
+  # either "efi" or "hybrid"
+  bootSize ? "256M"
+
+, # The files and directories to be placed in the target file system.
   # This is a list of attribute sets {source, target} where `source'
   # is the file system object (regular file or directory) to be
   # grafted in the file system at path `target'.
-, contents ? []
+  contents ? []
 
 , # Type of partition table to use; either "legacy", "efi", or "none".
   # For "efi" images, the GPT partition table is used and a mandatory ESP
   #   partition of reasonable size is created in addition to the root partition.
-  #   If `installBootLoader` is true, GRUB will be installed in EFI mode.
   # For "legacy", the msdos partition table is used and a single large root
-  #   partition is created. If `installBootLoader` is true, GRUB will be
-  #   installed in legacy mode.
+  #   partition is created.
+  # For "hybrid", the GPT partition table is used and a mandatory ESP
+  #   partition of reasonable size is created in addition to the root partition.
+  #   Also a legacy MBR will be present.
   # For "none", no partition table is created. Enabling `installBootLoader`
   #   most likely fails as GRUB will probably refuse to install.
   partitionTableType ? "legacy"
@@ -39,11 +50,11 @@
 
 , name ? "nixos-disk-image"
 
-, # Disk image format, one of qcow2, qcow2-compressed, vpc, raw.
+, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw.
   format ? "raw"
 }:
 
-assert partitionTableType == "legacy" || partitionTableType == "efi" || partitionTableType == "none";
+assert partitionTableType == "legacy" || partitionTableType == "efi" || partitionTableType == "hybrid" || partitionTableType == "none";
 # We use -E offset=X below, which is only supported by e2fsprogs
 assert partitionTableType != "none" -> fsType == "ext4";
 
@@ -57,13 +68,15 @@ let format' = format; in let
 
   filename = "nixos." + {
     qcow2 = "qcow2";
+    vdi   = "vdi";
     vpc   = "vhd";
     raw   = "img";
-  }.${format};
+  }.${format} or format;
 
   rootPartition = { # switch-case
     legacy = "1";
     efi = "2";
+    hybrid = "3";
   }.${partitionTableType};
 
   partitionDiskScript = { # switch-case
@@ -75,9 +88,18 @@ let format' = format; in let
     efi = ''
       parted --script $diskImage -- \
         mklabel gpt \
-        mkpart ESP fat32 8MiB 256MiB \
+        mkpart ESP fat32 8MiB ${bootSize} \
         set 1 boot on \
-        mkpart primary ext4 256MiB -1
+        mkpart primary ext4 ${bootSize} -1
+    '';
+    hybrid = ''
+      parted --script $diskImage -- \
+        mklabel gpt \
+        mkpart ESP fat32 8MiB ${bootSize} \
+        set 1 boot on \
+        mkpart no-fs 0 1024KiB \
+        set 2 bios_grub on \
+        mkpart primary ext4 ${bootSize} -1
     '';
     none = "";
   }.${partitionTableType};
@@ -128,19 +150,6 @@ let format' = format; in let
     }
 
     mkdir $out
-    diskImage=nixos.raw
-    truncate -s ${toString diskSize}M $diskImage
-
-    ${partitionDiskScript}
-
-    ${if partitionTableType != "none" then ''
-      # Get start & length of the root partition in sectors to $START and $SECTORS.
-      eval $(partx $diskImage -o START,SECTORS --nr ${rootPartition} --pairs)
-
-      mkfs.${fsType} -F -L ${label} $diskImage -E offset=$(sectorsToBytes $START) $(sectorsToKilobytes $SECTORS)K
-    '' else ''
-      mkfs.${fsType} -F -L ${label} $diskImage
-    ''}
 
     root="$PWD/root"
     mkdir -p $root
@@ -180,10 +189,36 @@ let format' = format; in let
     export NIX_STATE_DIR=$TMPDIR/state
     nix-store --load-db < ${closureInfo}/registration
 
+    chmod 755 "$TMPDIR"
     echo "running nixos-install..."
     nixos-install --root $root --no-bootloader --no-root-passwd \
       --system ${config.system.build.toplevel} --channel ${channelSources} --substituters ""
 
+    diskImage=nixos.raw
+
+    ${if diskSize == "auto" then ''
+      ${if partitionTableType == "efi" || partitionTableType == "hybrid" then ''
+        additionalSpace=$(( ($(numfmt --from=iec '${additionalSpace}') + $(numfmt --from=iec '${bootSize}')) / 1000 ))
+      '' else ''
+        additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') / 1000 ))
+      ''}
+      diskSize=$(( $(set -- $(du -d0 $root); echo "$1") + $additionalSpace ))
+      truncate -s "$diskSize"K $diskImage
+    '' else ''
+      truncate -s ${toString diskSize}M $diskImage
+    ''}
+
+    ${partitionDiskScript}
+
+    ${if partitionTableType != "none" then ''
+      # Get start & length of the root partition in sectors to $START and $SECTORS.
+      eval $(partx $diskImage -o START,SECTORS --nr ${rootPartition} --pairs)
+
+      mkfs.${fsType} -F -L ${label} $diskImage -E offset=$(sectorsToBytes $START) $(sectorsToKilobytes $SECTORS)K
+    '' else ''
+      mkfs.${fsType} -F -L ${label} $diskImage
+    ''}
+
     echo "copying staging root to image..."
     cptofs -p ${optionalString (partitionTableType != "none") "-P ${rootPartition}"} -t ${fsType} -i $diskImage $root/* /
   '';
@@ -217,7 +252,7 @@ in pkgs.vmTools.runInLinuxVM (
 
       # Create the ESP and mount it. Unlike e2fsprogs, mkfs.vfat doesn't support an
       # '-E offset=X' option, so we can't do this outside the VM.
-      ${optionalString (partitionTableType == "efi") ''
+      ${optionalString (partitionTableType == "efi" || partitionTableType == "hybrid") ''
         mkdir -p /mnt/boot
         mkfs.vfat -n ESP /dev/vda1
         mount /dev/vda1 /mnt/boot
diff --git a/nixpkgs/nixos/lib/make-ext4-fs.nix b/nixpkgs/nixos/lib/make-ext4-fs.nix
index 627ac324cf5..74a6c134e64 100644
--- a/nixpkgs/nixos/lib/make-ext4-fs.nix
+++ b/nixpkgs/nixos/lib/make-ext4-fs.nix
@@ -17,7 +17,7 @@
 , e2fsprogs
 , libfaketime
 , perl
-, lkl
+, fakeroot
 }:
 
 let
@@ -26,7 +26,7 @@ in
 pkgs.stdenv.mkDerivation {
   name = "ext4-fs.img${lib.optionalString compressImage ".zst"}";
 
-  nativeBuildInputs = [ e2fsprogs.bin libfaketime perl lkl ]
+  nativeBuildInputs = [ e2fsprogs.bin libfaketime perl fakeroot ]
   ++ lib.optional compressImage zstd;
 
   buildCommand =
@@ -37,32 +37,31 @@ pkgs.stdenv.mkDerivation {
       ${populateImageCommands}
       )
 
-      # Add the closures of the top-level store objects.
-      storePaths=$(cat ${sdClosureInfo}/store-paths)
+      echo "Preparing store paths for image..."
+
+      # Create nix/store before copying path
+      mkdir -p ./rootImage/nix/store
+
+      xargs -I % cp -a --reflink=auto % -t ./rootImage/nix/store/ < ${sdClosureInfo}/store-paths
+      (
+        GLOBIGNORE=".:.."
+        shopt -u dotglob
+        cp -a --reflink=auto ./files/* -t ./rootImage/
+      )
+
+      # Also include a manifest of the closures in a format suitable for nix-store --load-db
+      cp ${sdClosureInfo}/registration ./rootImage/nix-path-registration
 
       # Make a crude approximation of the size of the target image.
       # If the script starts failing, increase the fudge factors here.
-      numInodes=$(find $storePaths ./files | wc -l)
-      numDataBlocks=$(du -s -c -B 4096 --apparent-size $storePaths ./files | tail -1 | awk '{ print int($1 * 1.03) }')
+      numInodes=$(find ./rootImage | wc -l)
+      numDataBlocks=$(du -s -c -B 4096 --apparent-size ./rootImage | tail -1 | awk '{ print int($1 * 1.10) }')
       bytes=$((2 * 4096 * $numInodes + 4096 * $numDataBlocks))
       echo "Creating an EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks)"
 
       truncate -s $bytes $img
-      faketime -f "1970-01-01 00:00:01" mkfs.ext4 -L ${volumeLabel} -U ${uuid} $img
-
-      # Also include a manifest of the closures in a format suitable for nix-store --load-db.
-      cp ${sdClosureInfo}/registration nix-path-registration
-      cptofs -t ext4 -i $img nix-path-registration /
-
-      # Create nix/store before copying paths
-      faketime -f "1970-01-01 00:00:01" mkdir -p nix/store
-      cptofs -t ext4 -i $img nix /
-
-      echo "copying store paths to image..."
-      cptofs -t ext4 -i $img $storePaths /nix/store/
 
-      echo "copying files to image..."
-      cptofs -t ext4 -i $img ./files/* /
+      faketime -f "1970-01-01 00:00:01" fakeroot mkfs.ext4 -L ${volumeLabel} -U ${uuid} -d ./rootImage $img
 
       export EXT2FS_NO_MTAB_OK=yes
       # I have ended up with corrupted images sometimes, I suspect that happens when the build machine's disk gets full during the build.
diff --git a/nixpkgs/nixos/lib/make-iso9660-image.nix b/nixpkgs/nixos/lib/make-iso9660-image.nix
index 0f3f2b5b523..6a0e0e7c635 100644
--- a/nixpkgs/nixos/lib/make-iso9660-image.nix
+++ b/nixpkgs/nixos/lib/make-iso9660-image.nix
@@ -1,4 +1,4 @@
-{ stdenv, closureInfo, xorriso, syslinux
+{ stdenv, closureInfo, xorriso, syslinux, libossp_uuid
 
 , # The file name of the resulting ISO image.
   isoName ? "cd.iso"
@@ -34,8 +34,8 @@
 , # The path (outside the ISO file system) of the isohybrid-mbr image.
   isohybridMbrImage ? ""
 
-, # Whether to compress the resulting ISO image with bzip2.
-  compressImage ? false
+, # Whether to compress the resulting ISO image with zstd.
+  compressImage ? false, zstd
 
 , # The volume ID.
   volumeID ? ""
@@ -48,7 +48,7 @@ assert usbBootable -> isohybridMbrImage != "";
 stdenv.mkDerivation {
   name = isoName;
   builder = ./make-iso9660-image.sh;
-  buildInputs = [ xorriso syslinux ];
+  buildInputs = [ xorriso syslinux zstd libossp_uuid ];
 
   inherit isoName bootable bootImage compressImage volumeID efiBootImage efiBootable isohybridMbrImage usbBootable;
 
diff --git a/nixpkgs/nixos/lib/make-iso9660-image.sh b/nixpkgs/nixos/lib/make-iso9660-image.sh
index d4633d2c8d1..4740b05f955 100644
--- a/nixpkgs/nixos/lib/make-iso9660-image.sh
+++ b/nixpkgs/nixos/lib/make-iso9660-image.sh
@@ -99,7 +99,12 @@ done
 
 mkdir -p $out/iso
 
+# daed2280-b91e-42c0-aed6-82c825ca41f3 is an arbitrary namespace, to prevent
+# independent applications from generating the same UUID for the same value.
+# (the chance of that being problematic seem pretty slim here, but that's how
+# version-5 UUID's work)
 xorriso="xorriso
+ -boot_image any gpt_disk_guid=$(uuid -v 5 daed2280-b91e-42c0-aed6-82c825ca41f3 $out | tr -d -)
  -as mkisofs
  -iso-level 3
  -volid ${volumeID}
@@ -118,20 +123,16 @@ xorriso="xorriso
 
 $xorriso -output $out/iso/$isoName
 
-if test -n "$usbBootable"; then
-    echo "Making image hybrid..."
-    if test -n "$efiBootable"; then
-        isohybrid --uefi $out/iso/$isoName
-    else
-        isohybrid $out/iso/$isoName
-    fi
-fi
-
 if test -n "$compressImage"; then
     echo "Compressing image..."
-    bzip2 $out/iso/$isoName
+    zstd -T$NIX_BUILD_CORES --rm $out/iso/$isoName
 fi
 
 mkdir -p $out/nix-support
 echo $system > $out/nix-support/system
-echo "file iso $out/iso/$isoName" >> $out/nix-support/hydra-build-products
+
+if test -n "$compressImage"; then
+    echo "file iso $out/iso/$isoName.zst" >> $out/nix-support/hydra-build-products
+else
+    echo "file iso $out/iso/$isoName" >> $out/nix-support/hydra-build-products
+fi
diff --git a/nixpkgs/nixos/lib/make-options-doc/default.nix b/nixpkgs/nixos/lib/make-options-doc/default.nix
index 772b7d3add9..a1161621f0d 100644
--- a/nixpkgs/nixos/lib/make-options-doc/default.nix
+++ b/nixpkgs/nixos/lib/make-options-doc/default.nix
@@ -36,7 +36,7 @@ let
     // lib.optionalAttrs (opt ? example) { example = substFunction opt.example; }
     // lib.optionalAttrs (opt ? default) { default = substFunction opt.default; }
     // lib.optionalAttrs (opt ? type) { type = substFunction opt.type; }
-    // lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != []) { relatedPackages = genRelatedPackages opt.relatedPackages; }
+    // lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != []) { relatedPackages = genRelatedPackages opt.relatedPackages opt.name; }
    );
 
   # Generate DocBook documentation for a list of packages. This is
@@ -48,7 +48,7 @@ let
   # - a list:    that will be interpreted as an attribute path from `pkgs`,
   # - an attrset: that can specify `name`, `path`, `package`, `comment`
   #   (either of `name`, `path` is required, the rest are optional).
-  genRelatedPackages = packages:
+  genRelatedPackages = packages: optName:
     let
       unpack = p: if lib.isString p then { name = p; }
                   else if lib.isList p then { path = p; }
@@ -58,7 +58,7 @@ let
           title = args.title or null;
           name = args.name or (lib.concatStringsSep "." args.path);
           path = args.path or [ args.name ];
-          package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}'") pkgs);
+          package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}' found while evaluating `relatedPackages' of option `${optName}'") pkgs);
         in "<listitem>"
         + "<para><literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name} (${package.meta.name})</literal>"
         + lib.optionalString (!package.meta.available) " <emphasis>[UNAVAILABLE]</emphasis>"
diff --git a/nixpkgs/nixos/lib/qemu-flags.nix b/nixpkgs/nixos/lib/qemu-flags.nix
index 859d9e975fe..0cf6977af4b 100644
--- a/nixpkgs/nixos/lib/qemu-flags.nix
+++ b/nixpkgs/nixos/lib/qemu-flags.nix
@@ -2,13 +2,18 @@
 { pkgs }:
 
 let
-  zeroPad = n: if n < 10 then "0${toString n}" else toString n;
+  zeroPad = n:
+    pkgs.lib.optionalString (n < 16) "0" +
+      (if n > 255
+       then throw "Can't have more than 255 nets or nodes!"
+       else pkgs.lib.toHexString n);
 in
 
-{
+rec {
+  qemuNicMac = net: machine: "52:54:00:12:${zeroPad net}:${zeroPad machine}";
 
   qemuNICFlags = nic: net: machine:
-    [ "-device virtio-net-pci,netdev=vlan${toString nic},mac=52:54:00:12:${zeroPad net}:${zeroPad machine}"
+    [ "-device virtio-net-pci,netdev=vlan${toString nic},mac=${qemuNicMac net machine}"
       "-netdev vde,id=vlan${toString nic},sock=$QEMU_VDE_SOCKET_${toString net}"
     ];
 
diff --git a/nixpkgs/nixos/lib/test-driver/Logger.pm b/nixpkgs/nixos/lib/test-driver/Logger.pm
index 080310ea34e..a3384084a0e 100644
--- a/nixpkgs/nixos/lib/test-driver/Logger.pm
+++ b/nixpkgs/nixos/lib/test-driver/Logger.pm
@@ -8,17 +8,17 @@ use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 
 sub new {
     my ($class) = @_;
-    
+
     my $logFile = defined $ENV{LOGFILE} ? "$ENV{LOGFILE}" : "/dev/null";
     my $log = new XML::Writer(OUTPUT => new IO::File(">$logFile"));
-    
+
     my $self = {
         log => $log,
         logQueue => Thread::Queue->new()
     };
-    
+
     $self->{log}->startTag("logfile");
-    
+
     bless $self, $class;
     return $self;
 }
diff --git a/nixpkgs/nixos/lib/test-driver/test-driver.py b/nixpkgs/nixos/lib/test-driver/test-driver.py
index e7b05968b07..7b8d5803aa5 100644
--- a/nixpkgs/nixos/lib/test-driver/test-driver.py
+++ b/nixpkgs/nixos/lib/test-driver/test-driver.py
@@ -3,7 +3,10 @@ from contextlib import contextmanager, _GeneratorContextManager
 from queue import Queue, Empty
 from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List
 from xml.sax.saxutils import XMLGenerator
+import queue
+import io
 import _thread
+import argparse
 import atexit
 import base64
 import codecs
@@ -19,6 +22,7 @@ import subprocess
 import sys
 import tempfile
 import time
+import traceback
 import unicodedata
 
 CHAR_TO_KEY = {
@@ -671,6 +675,22 @@ class Machine:
         with self.nested("waiting for {} to appear on screen".format(regex)):
             retry(screen_matches)
 
+    def wait_for_console_text(self, regex: str) -> None:
+        self.log("waiting for {} to appear on console".format(regex))
+        # Buffer the console output, this is needed
+        # to match multiline regexes.
+        console = io.StringIO()
+        while True:
+            try:
+                console.write(self.last_lines.get())
+            except queue.Empty:
+                self.sleep(1)
+                continue
+            console.seek(0)
+            matches = re.search(regex, console.read())
+            if matches is not None:
+                return
+
     def send_key(self, key: str) -> None:
         key = CHAR_TO_KEY.get(key, key)
         self.send_monitor_command("sendkey {}".format(key))
@@ -734,11 +754,16 @@ class Machine:
         self.monitor, _ = self.monitor_socket.accept()
         self.shell, _ = self.shell_socket.accept()
 
+        # Store last serial console lines for use
+        # of wait_for_console_text
+        self.last_lines: Queue = Queue()
+
         def process_serial_output() -> None:
             assert self.process.stdout is not None
             for _line in self.process.stdout:
                 # Ignore undecodable bytes that may occur in boot menus
                 line = _line.decode(errors="ignore").replace("\r", "").rstrip()
+                self.last_lines.put(line)
                 eprint("{} # {}".format(self.name, line))
                 self.logger.enqueue({"msg": line, "machine": self.name})
 
@@ -751,6 +776,11 @@ class Machine:
 
         self.log("QEMU running (pid {})".format(self.pid))
 
+    def cleanup_statedir(self) -> None:
+        self.log("delete the VM state directory")
+        if os.path.isfile(self.state_dir):
+            shutil.rmtree(self.state_dir)
+
     def shutdown(self) -> None:
         if not self.booted:
             return
@@ -863,7 +893,8 @@ def run_tests() -> None:
             try:
                 exec(tests, globals())
             except Exception as e:
-                eprint("error: {}".format(str(e)))
+                eprint("error: ")
+                traceback.print_exc()
                 sys.exit(1)
     else:
         ptpython.repl.embed(locals(), globals())
@@ -889,6 +920,15 @@ def subtest(name: str) -> Iterator[None]:
 
 
 if __name__ == "__main__":
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument(
+        "-K",
+        "--keep-vm-state",
+        help="re-use a VM state coming from a previous run",
+        action="store_true",
+    )
+    (cli_args, vm_scripts) = arg_parser.parse_known_args()
+
     log = Logger()
 
     vlan_nrs = list(dict.fromkeys(os.environ.get("VLANS", "").split()))
@@ -896,8 +936,10 @@ if __name__ == "__main__":
     for nr, vde_socket, _, _ in vde_sockets:
         os.environ["QEMU_VDE_SOCKET_{}".format(nr)] = vde_socket
 
-    vm_scripts = sys.argv[1:]
     machines = [create_machine({"startCommand": s}) for s in vm_scripts]
+    for machine in machines:
+        if not cli_args.keep_vm_state:
+            machine.cleanup_statedir()
     machine_eval = [
         "{0} = machines[{1}]".format(m.name, idx) for idx, m in enumerate(machines)
     ]
@@ -911,7 +953,6 @@ if __name__ == "__main__":
                     continue
                 log.log("killing {} (pid {})".format(machine.name, machine.pid))
                 machine.process.kill()
-
             for _, _, process, _ in vde_sockets:
                 process.terminate()
         log.close()
diff --git a/nixpkgs/nixos/lib/testing-python.nix b/nixpkgs/nixos/lib/testing-python.nix
index 123323711a7..c6939c7d698 100644
--- a/nixpkgs/nixos/lib/testing-python.nix
+++ b/nixpkgs/nixos/lib/testing-python.nix
@@ -114,7 +114,7 @@ rec {
 
       imagemagick_tiff = imagemagick_light.override { inherit libtiff; };
 
-      # Generate onvenience wrappers for running the test driver
+      # Generate convenience wrappers for running the test driver
       # interactively with the specified network, and for starting the
       # VMs from the command line.
       driver = let warn = if skipLint then lib.warn "Linting is disabled!" else lib.id; in warn (runCommand testDriverName
diff --git a/nixpkgs/nixos/lib/utils.nix b/nixpkgs/nixos/lib/utils.nix
index 21f4c7c6988..543c8a8882e 100644
--- a/nixpkgs/nixos/lib/utils.nix
+++ b/nixpkgs/nixos/lib/utils.nix
@@ -2,9 +2,11 @@ pkgs: with pkgs.lib;
 
 rec {
 
-  # Check whenever fileSystem is needed for boot
-  fsNeededForBoot = fs: fs.neededForBoot
-                     || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ];
+  # Check whenever fileSystem is needed for boot.  NOTE: Make sure
+  # pathsNeededForBoot is closed under the parent relationship, i.e. if /a/b/c
+  # is in the list, put /a and /a/b in as well.
+  pathsNeededForBoot = [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ];
+  fsNeededForBoot = fs: fs.neededForBoot || elem fs.mountPoint pathsNeededForBoot;
 
   # Check whenever `b` depends on `a` as a fileSystem
   fsBefore = a: b: a.mountPoint == b.device