diff options
Diffstat (limited to 'infra/libkookie/nixpkgs/nixos/lib')
6 files changed, 170 insertions, 130 deletions
diff --git a/infra/libkookie/nixpkgs/nixos/lib/build-vms.nix b/infra/libkookie/nixpkgs/nixos/lib/build-vms.nix index b1575fc13bbd..ebbb0296bef6 100644 --- a/infra/libkookie/nixpkgs/nixos/lib/build-vms.nix +++ b/infra/libkookie/nixpkgs/nixos/lib/build-vms.nix @@ -18,9 +18,6 @@ rec { inherit pkgs; - qemu = pkgs.qemu_test; - - # Build a virtual network from an attribute set `{ machine1 = # config1; ... machineN = configN; }', where `machineX' is the # hostname and `configX' is a NixOS system configuration. Each @@ -39,7 +36,6 @@ rec { [ ../modules/virtualisation/qemu-vm.nix ../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs { key = "no-manual"; documentation.nixos.enable = false; } - { key = "qemu"; system.build.qemu = qemu; } { key = "nodes"; _module.args.nodes = nodes; } ] ++ optional minimal ../modules/testing/minimal-kernel.nix; }; diff --git a/infra/libkookie/nixpkgs/nixos/lib/make-disk-image.nix b/infra/libkookie/nixpkgs/nixos/lib/make-disk-image.nix index 8aa606a56af8..0ad0cf1fef5a 100644 --- a/infra/libkookie/nixpkgs/nixos/lib/make-disk-image.nix +++ b/infra/libkookie/nixpkgs/nixos/lib/make-disk-image.nix @@ -28,6 +28,9 @@ # partition of reasonable size is created in addition to the root partition. # For "legacy", the msdos partition table is used and a single large root # partition is created. + # For "legacy+gpt", the GPT partition table is used, a 1MiB no-fs partition for + # use by the bootloader is created, and a single large root partition is + # created. # For "hybrid", the GPT partition table is used and a mandatory ESP # partition of reasonable size is created in addition to the root partition. # Also a legacy MBR will be present. @@ -54,7 +57,7 @@ format ? "raw" }: -assert partitionTableType == "legacy" || partitionTableType == "efi" || partitionTableType == "hybrid" || partitionTableType == "none"; +assert partitionTableType == "legacy" || partitionTableType == "legacy+gpt" || partitionTableType == "efi" || partitionTableType == "hybrid" || partitionTableType == "none"; # We use -E offset=X below, which is only supported by e2fsprogs assert partitionTableType != "none" -> fsType == "ext4"; @@ -75,6 +78,7 @@ let format' = format; in let rootPartition = { # switch-case legacy = "1"; + "legacy+gpt" = "2"; efi = "2"; hybrid = "3"; }.${partitionTableType}; @@ -85,6 +89,16 @@ let format' = format; in let mklabel msdos \ mkpart primary ext4 1MiB -1 ''; + "legacy+gpt" = '' + parted --script $diskImage -- \ + mklabel gpt \ + mkpart no-fs 1MB 2MB \ + set 1 bios_grub on \ + align-check optimal 1 \ + mkpart primary ext4 2MB -1 \ + align-check optimal 2 \ + print + ''; efi = '' parted --script $diskImage -- \ mklabel gpt \ @@ -120,7 +134,7 @@ let format' = format; in let binPath = with pkgs; makeBinPath ( [ rsync - utillinux + util-linux parted e2fsprogs lkl @@ -225,7 +239,7 @@ let format' = format; in let in pkgs.vmTools.runInLinuxVM ( pkgs.runCommand name { preVM = prepareImage; - buildInputs = with pkgs; [ utillinux e2fsprogs dosfstools ]; + buildInputs = with pkgs; [ util-linux e2fsprogs dosfstools ]; postVM = '' ${if format == "raw" then '' mv $diskImage $out/${filename} diff --git a/infra/libkookie/nixpkgs/nixos/lib/make-iso9660-image.nix b/infra/libkookie/nixpkgs/nixos/lib/make-iso9660-image.nix index 6a0e0e7c635a..549530965f6e 100644 --- a/infra/libkookie/nixpkgs/nixos/lib/make-iso9660-image.nix +++ b/infra/libkookie/nixpkgs/nixos/lib/make-iso9660-image.nix @@ -48,7 +48,7 @@ assert usbBootable -> isohybridMbrImage != ""; stdenv.mkDerivation { name = isoName; builder = ./make-iso9660-image.sh; - buildInputs = [ xorriso syslinux zstd libossp_uuid ]; + nativeBuildInputs = [ xorriso syslinux zstd libossp_uuid ]; inherit isoName bootable bootImage compressImage volumeID efiBootImage efiBootable isohybridMbrImage usbBootable; diff --git a/infra/libkookie/nixpkgs/nixos/lib/make-system-tarball.nix b/infra/libkookie/nixpkgs/nixos/lib/make-system-tarball.nix index dee91a6ce3f4..dab168f4a481 100644 --- a/infra/libkookie/nixpkgs/nixos/lib/make-system-tarball.nix +++ b/infra/libkookie/nixpkgs/nixos/lib/make-system-tarball.nix @@ -37,7 +37,7 @@ in stdenv.mkDerivation { name = "tarball"; builder = ./make-system-tarball.sh; - buildInputs = extraInputs; + nativeBuildInputs = extraInputs; inherit fileName extraArgs extraCommands compressCommand; diff --git a/infra/libkookie/nixpkgs/nixos/lib/test-driver/test-driver.py b/infra/libkookie/nixpkgs/nixos/lib/test-driver/test-driver.py index 156392ad1e32..96b75a49928f 100644 --- a/infra/libkookie/nixpkgs/nixos/lib/test-driver/test-driver.py +++ b/infra/libkookie/nixpkgs/nixos/lib/test-driver/test-driver.py @@ -110,7 +110,6 @@ def create_vlan(vlan_nr: str) -> Tuple[str, str, "subprocess.Popen[bytes]", Any] pty_master, pty_slave = pty.openpty() vde_process = subprocess.Popen( ["vde_switch", "-s", vde_socket, "--dirmode", "0700"], - bufsize=1, stdin=pty_slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -635,8 +634,7 @@ class Machine: shutil.copy(intermediate, abs_target) def dump_tty_contents(self, tty: str) -> None: - """Debugging: Dump the contents of the TTY<n> - """ + """Debugging: Dump the contents of the TTY<n>""" self.execute("fold -w 80 /dev/vcs{} | systemd-cat".format(tty)) def get_screen_text(self) -> str: @@ -748,7 +746,6 @@ class Machine: self.process = subprocess.Popen( self.script, - bufsize=1, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -862,8 +859,7 @@ class Machine: self.send_monitor_command("set_link virtio-net-pci.1 off") def unblock(self) -> None: - """Make the machine reachable. - """ + """Make the machine reachable.""" self.send_monitor_command("set_link virtio-net-pci.1 on") diff --git a/infra/libkookie/nixpkgs/nixos/lib/testing-python.nix b/infra/libkookie/nixpkgs/nixos/lib/testing-python.nix index 498f97336c02..13abfb9a111d 100644 --- a/infra/libkookie/nixpkgs/nixos/lib/testing-python.nix +++ b/infra/libkookie/nixpkgs/nixos/lib/testing-python.nix @@ -3,13 +3,13 @@ # Use a minimal kernel? , minimal ? false # Ignored -, config ? {} +, config ? { } # !!! See comment about args in lib/modules.nix -, specialArgs ? {} +, specialArgs ? { } # Modules to add to each VM -, extraConfigurations ? [] }: +, extraConfigurations ? [ ] +}: -with import ./build-vms.nix { inherit system pkgs minimal specialArgs extraConfigurations; }; with pkgs; rec { @@ -17,40 +17,41 @@ rec { inherit pkgs; - testDriver = let - testDriverScript = ./test-driver/test-driver.py; - in stdenv.mkDerivation { - name = "nixos-test-driver"; - - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ]; - checkInputs = with python3Packages; [ pylint black mypy ]; - - dontUnpack = true; + mkTestDriver = + let + testDriverScript = ./test-driver/test-driver.py; + in + qemu_pkg: stdenv.mkDerivation { + name = "nixos-test-driver"; - preferLocalBuild = true; + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ]; + checkInputs = with python3Packages; [ pylint black mypy ]; - doCheck = true; - checkPhase = '' - mypy --disallow-untyped-defs \ - --no-implicit-optional \ - --ignore-missing-imports ${testDriverScript} - pylint --errors-only ${testDriverScript} - black --check --diff ${testDriverScript} - ''; + dontUnpack = true; - installPhase = - '' - mkdir -p $out/bin - cp ${testDriverScript} $out/bin/nixos-test-driver - chmod u+x $out/bin/nixos-test-driver - # TODO: copy user script part into this file (append) + preferLocalBuild = true; - wrapProgram $out/bin/nixos-test-driver \ - --prefix PATH : "${lib.makeBinPath [ qemu_test vde2 netpbm coreutils ]}" \ + doCheck = true; + checkPhase = '' + mypy --disallow-untyped-defs \ + --no-implicit-optional \ + --ignore-missing-imports ${testDriverScript} + pylint --errors-only ${testDriverScript} + black --check --diff ${testDriverScript} ''; - }; + installPhase = + '' + mkdir -p $out/bin + cp ${testDriverScript} $out/bin/nixos-test-driver + chmod u+x $out/bin/nixos-test-driver + # TODO: copy user script part into this file (append) + + wrapProgram $out/bin/nixos-test-driver \ + --prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \ + ''; + }; # Run an automated test suite in the given virtual network. # `driver' is the script that runs the network. @@ -73,11 +74,10 @@ rec { { testScript , enableOCR ? false , name ? "unnamed" - # Skip linting (mainly intended for faster dev cycles) + # Skip linting (mainly intended for faster dev cycles) , skipLint ? false , ... } @ t: - let # A standard store path to the vm monitor is built like this: # /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor @@ -86,25 +86,7 @@ rec { maxTestNameLen = 50; testNameLen = builtins.stringLength name; - testDriverName = with builtins; - if testNameLen > maxTestNameLen then - abort ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " + - "it's currently ${toString testNameLen} characters long.") - else - "nixos-test-driver-${name}"; - nodes = buildVirtualNetwork ( - t.nodes or (if t ? machine then { machine = t.machine; } else { })); - - testScript' = - # Call the test script with the computed nodes. - if lib.isFunction testScript - then testScript { inherit nodes; } - else testScript; - - vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes); - - vms = map (m: m.config.system.build.vm) (lib.attrValues nodes); ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; }; @@ -113,72 +95,124 @@ rec { # Generate convenience wrappers for running the test driver # interactively with the specified network, and for starting the # VMs from the command line. - driver = let warn = if skipLint then lib.warn "Linting is disabled!" else lib.id; in warn (runCommand testDriverName - { buildInputs = [ makeWrapper]; - testScript = testScript'; - preferLocalBuild = true; - testName = name; - } - '' - mkdir -p $out/bin - - echo -n "$testScript" > $out/test-script - ${lib.optionalString (!skipLint) '' - ${python3Packages.black}/bin/black --check --diff $out/test-script - ''} - - ln -s ${testDriver}/bin/nixos-test-driver $out/bin/ - vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) - wrapProgram $out/bin/nixos-test-driver \ - --add-flags "''${vms[*]}" \ - ${lib.optionalString enableOCR - "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \ - --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \ - --set VLANS '${toString vlans}' - ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms - wrapProgram $out/bin/nixos-run-vms \ - --add-flags "''${vms[*]}" \ - ${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \ - --set tests 'start_all(); join_all();' \ - --set VLANS '${toString vlans}' \ - ${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"} - ''); # " + mkDriver = qemu_pkg: + let + build-vms = import ./build-vms.nix { + inherit system pkgs minimal specialArgs; + extraConfigurations = extraConfigurations ++ (pkgs.lib.optional (qemu_pkg != null) + { + virtualisation.qemu.package = qemu_pkg; + } + ); + }; + + # FIXME: get this pkg from the module system + testDriver = mkTestDriver (if qemu_pkg == null then pkgs.qemu_test else qemu_pkg); + + nodes = build-vms.buildVirtualNetwork ( + t.nodes or (if t ? machine then { machine = t.machine; } else { }) + ); + vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes); + vms = map (m: m.config.system.build.vm) (lib.attrValues nodes); + + testScript' = + # Call the test script with the computed nodes. + if lib.isFunction testScript + then testScript { inherit nodes; } + else testScript; + + testDriverName = with builtins; + if testNameLen > maxTestNameLen then + abort + ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " + + "it's currently ${toString testNameLen} characters long.") + else + "nixos-test-driver-${name}"; + + warn = if skipLint then lib.warn "Linting is disabled!" else lib.id; + in + warn (runCommand testDriverName + { + buildInputs = [ makeWrapper ]; + testScript = testScript'; + preferLocalBuild = true; + testName = name; + passthru = { + inherit nodes; + }; + } + '' + mkdir -p $out/bin + + echo -n "$testScript" > $out/test-script + ${lib.optionalString (!skipLint) '' + ${python3Packages.black}/bin/black --check --diff $out/test-script + ''} + + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/ + vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) + wrapProgram $out/bin/nixos-test-driver \ + --add-flags "''${vms[*]}" \ + ${lib.optionalString enableOCR + "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \ + --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \ + --set VLANS '${toString vlans}' + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms + wrapProgram $out/bin/nixos-run-vms \ + --add-flags "''${vms[*]}" \ + ${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \ + --set tests 'start_all(); join_all();' \ + --set VLANS '${toString vlans}' \ + ${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"} + ''); # " passMeta = drv: drv // lib.optionalAttrs (t ? meta) { - meta = (drv.meta or {}) // t.meta; + meta = (drv.meta or { }) // t.meta; }; + driver = mkDriver null; + driverInteractive = mkDriver pkgs.qemu; + test = passMeta (runTests driver); - nodeNames = builtins.attrNames nodes; + nodeNames = builtins.attrNames driver.nodes; invalidNodeNames = lib.filter - (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) nodeNames; + (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) + nodeNames; in - if lib.length invalidNodeNames > 0 then - throw '' - Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! - All machines are referenced as python variables in the testing framework which will break the - script when special characters are used. + if lib.length invalidNodeNames > 0 then + throw '' + Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! + All machines are referenced as python variables in the testing framework which will break the + script when special characters are used. - Please stick to alphanumeric chars and underscores as separation. - '' - else - test // { - inherit nodes driver test; - }; + Please stick to alphanumeric chars and underscores as separation. + '' + else + test // { + inherit test driver driverInteractive; + inherit (driver) nodes; + }; runInMachine = { drv , machine , preBuild ? "" , postBuild ? "" + , qemu ? pkgs.qemu_test , ... # ??? }: let - vm = buildVM { } - [ machine - { key = "run-in-machine"; + build-vms = import ./build-vms.nix { + inherit system pkgs minimal specialArgs extraConfigurations; + }; + + vm = build-vms.buildVM { } + [ + machine + { + key = "run-in-machine"; networking.hostName = "client"; nix.readOnlyStore = false; virtualisation.writableStore = false; @@ -221,20 +255,20 @@ rec { unset xchg export tests='${testScript}' - ${testDriver}/bin/nixos-test-driver ${vm.config.system.build.vm}/bin/run-*-vm + ${mkTestDriver qemu}/bin/nixos-test-driver --keep-vm-state ${vm.config.system.build.vm}/bin/run-*-vm ''; # */ in - lib.overrideDerivation drv (attrs: { - requiredSystemFeatures = [ "kvm" ]; - builder = "${bash}/bin/sh"; - args = ["-e" vmRunCommand]; - origArgs = attrs.args; - origBuilder = attrs.builder; - }); + lib.overrideDerivation drv (attrs: { + requiredSystemFeatures = [ "kvm" ]; + builder = "${bash}/bin/sh"; + args = [ "-e" vmRunCommand ]; + origArgs = attrs.args; + origBuilder = attrs.builder; + }); - runInMachineWithX = { require ? [], ... } @ args: + runInMachineWithX = { require ? [ ], ... } @ args: let client = { ... }: @@ -250,13 +284,13 @@ rec { services.xserver.windowManager.icewm.enable = true; }; in - runInMachine ({ - machine = client; - preBuild = - '' - client.wait_for_x() - ''; - } // args); + runInMachine ({ + machine = client; + preBuild = + '' + client.wait_for_x() + ''; + } // args); simpleTest = as: (makeTest as).test; |