From b3de22483cfd40e52dbe6fe7317b0f4901bd957d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 6 Jun 2022 13:29:04 +0200 Subject: [PATCH] nixos/testing-python.nix: Add evalTest This is a decomposition of the testing-python.nix and build-vms.nix files into modules. By refactoring the glue, we accomplish the following: - NixOS tests can now use `imports` and other module system features. - Network-wide test setup can now be reusable; example: - A setup with all VMs configured to use a DNS server - Split long, slow tests into multiple tests that import a common module that has most of the setup. - Type checking for the test arguments - (TBD) "generated" options reference docs - Aspects that had to be wired through all the glue are now in their own files. - Chief example: interactive.nix. - Also: network.nix In rewriting this, I've generally stuck as close as possible to the existing code; copying pieces of logic and rewiring them, without changing the logic itself. I've made two exceptions to this rule - Introduction of `extraDriverArgs` instead of hardcoded interactivity logic. - Incorporation of https://github.com/NixOS/nixpkgs/pull/144110 in testScript.nix. I might revert the latter and split it into a new commit. --- nixos/lib/testing-python.nix | 20 ++++ nixos/lib/testing/call-test.nix | 16 +++ nixos/lib/testing/driver.nix | 177 ++++++++++++++++++++++++++++++ nixos/lib/testing/interactive.nix | 18 +++ nixos/lib/testing/legacy.nix | 25 +++++ nixos/lib/testing/meta.nix | 12 ++ nixos/lib/testing/name.nix | 7 ++ nixos/lib/testing/network.nix | 75 +++++++++++++ nixos/lib/testing/nodes.nix | 91 +++++++++++++++ nixos/lib/testing/run.nix | 51 +++++++++ nixos/lib/testing/testScript.nix | 78 +++++++++++++ 11 files changed, 570 insertions(+) create mode 100644 nixos/lib/testing/call-test.nix create mode 100644 nixos/lib/testing/driver.nix create mode 100644 nixos/lib/testing/interactive.nix create mode 100644 nixos/lib/testing/legacy.nix create mode 100644 nixos/lib/testing/meta.nix create mode 100644 nixos/lib/testing/name.nix create mode 100644 nixos/lib/testing/network.nix create mode 100644 nixos/lib/testing/nodes.nix create mode 100644 nixos/lib/testing/run.nix create mode 100644 nixos/lib/testing/testScript.nix diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 4bb1689ffd78..ab509c098d24 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -166,6 +166,26 @@ rec { ${lib.optionalString (interactive) "--add-flags --interactive"} ''); + evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; }; + runTest = module: (evalTest module).config.run; + + testModules = [ + ./testing/driver.nix + ./testing/interactive.nix + ./testing/legacy.nix + ./testing/meta.nix + ./testing/name.nix + ./testing/network.nix + ./testing/nodes.nix + ./testing/run.nix + ./testing/testScript.nix + { + config = { + hostPkgs = pkgs; + }; + } + ]; + # Make a full-blown test makeTest = { machine ? null diff --git a/nixos/lib/testing/call-test.nix b/nixos/lib/testing/call-test.nix new file mode 100644 index 000000000000..3e137e78cd47 --- /dev/null +++ b/nixos/lib/testing/call-test.nix @@ -0,0 +1,16 @@ +{ config, lib, ... }: +let + inherit (lib) mkOption types; +in +{ + options = { + callTest = mkOption { + internal = true; + type = types.functionTo types.raw; + }; + result = mkOption { + internal = true; + default = config; + }; + }; +} diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix new file mode 100644 index 000000000000..9473d888cbb8 --- /dev/null +++ b/nixos/lib/testing/driver.nix @@ -0,0 +1,177 @@ +{ config, lib, hostPkgs, ... }: +let + inherit (lib) mkOption types; + + # Reifies and correctly wraps the python test driver for + # the respective qemu version and with or without ocr support + testDriver = hostPkgs.callPackage ../test-driver { + inherit (config) enableOCR extraPythonPackages; + qemu_pkg = config.qemu.package; + imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; }; + tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; }; + }; + + + vlans = map (m: m.virtualisation.vlans) (lib.attrValues config.nodes); + vms = map (m: m.system.build.vm) (lib.attrValues config.nodes); + + nodeHostNames = + let + nodesList = map (c: c.system.name) (lib.attrValues config.nodes); + in + nodesList ++ lib.optional (lib.length nodesList == 1 && !lib.elem "machine" nodesList) "machine"; + + # TODO: This is an implementation error and needs fixing + # the testing famework cannot legitimately restrict hostnames further + # beyond RFC1035 + invalidNodeNames = lib.filter + (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) + nodeHostNames; + + uniqueVlans = lib.unique (builtins.concatLists vlans); + vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans; + machineNames = map (name: "${name}: Machine;") nodeHostNames; + + withChecks = + if lib.length invalidNodeNames > 0 then + throw '' + Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! + All machines are referenced as python variables in the testing framework which will break the + script when special characters are used. + + This is an IMPLEMENTATION ERROR and needs to be fixed. Meanwhile, + please stick to alphanumeric chars and underscores as separation. + '' + else + lib.warnIf config.skipLint "Linting is disabled"; + + driver = + hostPkgs.runCommand "nixos-test-driver-${config.name}" + { + # inherit testName; TODO (roberth): need this? + nativeBuildInputs = [ + hostPkgs.makeWrapper + ] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ]; + testScript = config.testScriptString; + preferLocalBuild = true; + passthru = config.passthru; + meta = config.meta // { + mainProgram = "nixos-test-driver"; + }; + } + '' + mkdir -p $out/bin + + vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) + + ${lib.optionalString (!config.skipTypeCheck) '' + # prepend type hints so the test script can be type checked with mypy + cat "${../test-script-prepend.py}" >> testScriptWithTypes + echo "${builtins.toString machineNames}" >> testScriptWithTypes + echo "${builtins.toString vlanNames}" >> testScriptWithTypes + echo -n "$testScript" >> testScriptWithTypes + + cat -n testScriptWithTypes + + # set pythonpath so mypy knows where to find the imports. this requires the py.typed file. + export PYTHONPATH='${../test-driver}' + mypy --no-implicit-optional \ + --pretty \ + --no-color-output \ + testScriptWithTypes + unset PYTHONPATH + ''} + + echo -n "$testScript" >> $out/test-script + + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver + + ${testDriver}/bin/generate-driver-symbols + ${lib.optionalString (!config.skipLint) '' + PYFLAKES_BUILTINS="$( + echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)}, + < ${lib.escapeShellArg "driver-symbols"} + )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script + ''} + + # set defaults through environment + # see: ./test-driver/test-driver.py argparse implementation + wrapProgram $out/bin/nixos-test-driver \ + --set startScripts "''${vmStartScripts[*]}" \ + --set testScript "$out/test-script" \ + --set vlans '${toString vlans}' \ + ${lib.escapeShellArgs (lib.concatMap (arg: ["--add-flags" arg]) config.extraDriverArgs)} + ''; + +in +{ + options = { + + driver = mkOption { + description = "Script that runs the test."; + type = types.package; + defaultText = lib.literalDocBook "set by the test framework"; + }; + + hostPkgs = mkOption { + description = "Nixpkgs attrset used outside the nodes."; + type = types.raw; + example = lib.literalExpression '' + import nixpkgs { inherit system config overlays; } + ''; + }; + + qemu.package = mkOption { + description = "Which qemu package to use."; + type = types.package; + default = hostPkgs.qemu_test; + defaultText = "hostPkgs.qemu_test"; + }; + + enableOCR = mkOption { + description = '' + Whether to enable Optical Character Recognition functionality for + testing graphical programs. + ''; + type = types.bool; + default = false; + }; + + extraPythonPackages = mkOption { + description = '' + Python packages to add to the test driver. + + The argument is a Python package set, similar to `pkgs.pythonPackages`. + ''; + type = types.functionTo (types.listOf types.package); + default = ps: [ ]; + }; + + extraDriverArgs = mkOption { + description = '' + Extra arguments to pass to the test driver. + ''; + type = types.listOf types.str; + default = []; + }; + + skipLint = mkOption { + type = types.bool; + default = false; + }; + + skipTypeCheck = mkOption { + type = types.bool; + default = false; + }; + }; + + config = { + _module.args.hostPkgs = config.hostPkgs; + + driver = withChecks driver; + + # make available on the test runner + passthru.driver = config.driver; + }; +} diff --git a/nixos/lib/testing/interactive.nix b/nixos/lib/testing/interactive.nix new file mode 100644 index 000000000000..fd4d481a3f82 --- /dev/null +++ b/nixos/lib/testing/interactive.nix @@ -0,0 +1,18 @@ +{ config, lib, moduleType, hostPkgs, ... }: +let + inherit (lib) mkOption types; +in +{ + options = { + interactive = mkOption { + description = "All the same options, but configured for interactive use."; + type = moduleType; + }; + }; + + config = { + interactive.qemu.package = hostPkgs.qemu; + interactive.extraDriverArgs = [ "--interactive" ]; + passthru.driverInteractive = config.interactive.driver; + }; +} diff --git a/nixos/lib/testing/legacy.nix b/nixos/lib/testing/legacy.nix new file mode 100644 index 000000000000..868b8b65b17d --- /dev/null +++ b/nixos/lib/testing/legacy.nix @@ -0,0 +1,25 @@ +{ config, options, lib, ... }: +let + inherit (lib) mkIf mkOption types; +in +{ + # This needs options.warnings, which we don't have (yet?). + # imports = [ + # (lib.mkRenamedOptionModule [ "machine" ] [ "nodes" "machine" ]) + # ]; + + options = { + machine = mkOption { + internal = true; + type = types.raw; + }; + }; + + config = { + nodes = mkIf options.machine.isDefined ( + lib.warn + "In test `${config.name}': The `machine' attribute in NixOS tests (pkgs.nixosTest / make-test-python.nix / testing-python.nix / makeTest) is deprecated. Please set the equivalent `nodes.machine'." + { inherit (config) machine; } + ); + }; +} diff --git a/nixos/lib/testing/meta.nix b/nixos/lib/testing/meta.nix new file mode 100644 index 000000000000..1312d6a986ec --- /dev/null +++ b/nixos/lib/testing/meta.nix @@ -0,0 +1,12 @@ +{ lib, ... }: +let + inherit (lib) types mkOption; +in +{ + options = { + meta.maintainers = lib.mkOption { + type = types.listOf types.raw; + default = []; + }; + }; +} diff --git a/nixos/lib/testing/name.nix b/nixos/lib/testing/name.nix new file mode 100644 index 000000000000..f9fa488511c0 --- /dev/null +++ b/nixos/lib/testing/name.nix @@ -0,0 +1,7 @@ +{ lib, ... }: +{ + options.name = lib.mkOption { + description = "The name of the test."; + type = lib.types.str; + }; +} diff --git a/nixos/lib/testing/network.nix b/nixos/lib/testing/network.nix new file mode 100644 index 000000000000..e0446846fb09 --- /dev/null +++ b/nixos/lib/testing/network.nix @@ -0,0 +1,75 @@ +{ lib, nodes, ... }: + +with lib; + + +let + machines = attrNames nodes; + + machinesNumbered = zipLists machines (range 1 254); + + nodes_ = forEach machinesNumbered (m: nameValuePair m.fst + [ + ({ config, nodes, pkgs, ... }: + let + interfacesNumbered = zipLists config.virtualisation.vlans (range 1 255); + interfaces = forEach interfacesNumbered ({ fst, snd }: + nameValuePair "eth${toString snd}" { + ipv4.addresses = + [{ + address = "192.168.${toString fst}.${toString m.snd}"; + prefixLength = 24; + }]; + }); + + networkConfig = + { + networking.hostName = mkDefault m.fst; + + networking.interfaces = listToAttrs interfaces; + + networking.primaryIPAddress = + optionalString (interfaces != [ ]) (head (head interfaces).value.ipv4.addresses).address; + + # Put the IP addresses of all VMs in this machine's + # /etc/hosts file. If a machine has multiple + # interfaces, use the IP address corresponding to + # the first interface (i.e. the first network in its + # virtualisation.vlans option). + networking.extraHosts = flip concatMapStrings machines + (m': + let config = (getAttr m' nodes).config; in + optionalString (config.networking.primaryIPAddress != "") + ("${config.networking.primaryIPAddress} " + + optionalString (config.networking.domain != null) + "${config.networking.hostName}.${config.networking.domain} " + + "${config.networking.hostName}\n")); + + virtualisation.qemu.options = + let qemu-common = import ../qemu-common.nix { inherit lib pkgs; }; + in + flip concatMap interfacesNumbered + ({ fst, snd }: qemu-common.qemuNICFlags snd fst m.snd); + }; + + in + { + key = "ip-address"; + config = networkConfig // { + # Expose the networkConfig items for tests like nixops + # that need to recreate the network config. + system.build.networkConfig = networkConfig; + }; + } + ) + ]); + + extraNodeConfigs = lib.listToAttrs nodes_; +in +{ + config = { + defaults = { config, name, ... }: { + imports = extraNodeConfigs.${name}; + }; + }; +} diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix new file mode 100644 index 000000000000..98580d5dc4f8 --- /dev/null +++ b/nixos/lib/testing/nodes.nix @@ -0,0 +1,91 @@ +testModuleArgs@{ config, lib, hostPkgs, nodes, ... }: + +let + inherit (lib) mkOption mkForce optional types mapAttrs mkDefault; + + system = hostPkgs.stdenv.hostPlatform.system; + + baseOS = + import ../eval-config.nix { + inherit system; + inherit (config.node) specialArgs; + modules = [ config.defaults ]; + baseModules = (import ../../modules/module-list.nix) ++ + [ + ../../modules/virtualisation/qemu-vm.nix + ../../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs + { key = "no-manual"; documentation.nixos.enable = false; } + { + key = "no-revision"; + # Make the revision metadata constant, in order to avoid needless retesting. + # The human version (e.g. 21.05-pre) is left as is, because it is useful + # for external modules that test with e.g. testers.nixosTest and rely on that + # version number. + config.system.nixos.revision = mkForce "constant-nixos-revision"; + } + { key = "nodes"; _module.args.nodes = nodes; } + + ({ config, ... }: + { + virtualisation.qemu.package = testModuleArgs.config.qemu.package; + + # Ensure we do not use aliases. Ideally this is only set + # when the test framework is used by Nixpkgs NixOS tests. + nixpkgs.config.allowAliases = false; + }) + ] ++ optional config.minimal ../../modules/testing/minimal-kernel.nix; + }; + + +in + +{ + + options = { + node.type = mkOption { + type = types.raw; + default = baseOS.type; + internal = true; + }; + + nodes = mkOption { + type = types.lazyAttrsOf config.node.type; + }; + + defaults = mkOption { + description = '' + NixOS configuration that is applied to all {option}`nodes`. + ''; + type = types.deferredModule; + default = { }; + }; + + node.specialArgs = mkOption { + type = types.lazyAttrsOf types.raw; + default = { }; + }; + + minimal = mkOption { + type = types.bool; + default = false; + }; + + nodesCompat = mkOption { + internal = true; + }; + }; + + config = { + _module.args.nodes = config.nodesCompat; + nodesCompat = + mapAttrs + (name: config: config // { + config = lib.warn + "Module argument `nodes.${name}.config` is deprecated. Use `nodes.${name}` instead." + config; + }) + config.nodes; + + passthru.nodes = config.nodesCompat; + }; +} diff --git a/nixos/lib/testing/run.nix b/nixos/lib/testing/run.nix new file mode 100644 index 000000000000..65bcbe720bf3 --- /dev/null +++ b/nixos/lib/testing/run.nix @@ -0,0 +1,51 @@ +{ config, hostPkgs, lib, ... }: +let + inherit (lib) types mkOption; +in +{ + options = { + passthru = mkOption { + type = types.lazyAttrsOf types.raw; + description = '' + Attributes to add to the returned derivations, + which are not necessarily part of the build. + + This is a bit like doing `drv // { myAttr = true; }` (which would be lost by `overrideAttrs`). + It does not change the actual derivation, but adds the attribute nonetheless, so that + consumers of what would be `drv` have more information. + ''; + }; + + run = mkOption { + type = types.package; + description = '' + Derivation that runs the test. + ''; + }; + }; + + config = { + run = hostPkgs.stdenv.mkDerivation { + name = "vm-test-run-${config.name}"; + + requiredSystemFeatures = [ "kvm" "nixos-test" ]; + + buildCommand = + '' + mkdir -p $out + + # effectively mute the XMLLogger + export LOGFILE=/dev/null + + ${config.driver}/bin/nixos-test-driver -o $out + ''; + + passthru = config.passthru; + + meta = config.meta; + }; + + # useful for inspection (debugging / exploration) + passthru.config = config; + }; +} diff --git a/nixos/lib/testing/testScript.nix b/nixos/lib/testing/testScript.nix new file mode 100644 index 000000000000..08e87b626b3b --- /dev/null +++ b/nixos/lib/testing/testScript.nix @@ -0,0 +1,78 @@ +testModuleArgs@{ config, lib, hostPkgs, nodes, moduleType, ... }: +let + inherit (lib) mkOption types; + inherit (types) either str functionTo; +in +{ + options = { + testScript = mkOption { + type = either str (functionTo str); + }; + testScriptString = mkOption { + type = str; + readOnly = true; + internal = true; + }; + + includeTestScriptReferences = mkOption { + type = types.bool; + default = true; + internal = true; + }; + withoutTestScriptReferences = mkOption { + type = moduleType; + description = '' + A parallel universe where the testScript is invalid and has no references. + ''; + }; + }; + config = { + withoutTestScriptReferences.includeTestScriptReferences = false; + withoutTestScriptReferences.testScript = lib.mkForce "testscript omitted"; + + testScriptString = + if lib.isFunction config.testScript + then + config.testScript + { + nodes = + lib.mapAttrs + (k: v: + if v.virtualisation.useNixStoreImage + then + # prevent infinite recursion when testScript would + # reference v's toplevel + config.withoutTestScriptReferences.nodesCompat.${k} + else + # reuse memoized config + v + ) + config.nodesCompat; + } + else config.testScript; + + defaults = { config, name, ... }: { + # Make sure all derivations referenced by the test + # script are available on the nodes. When the store is + # accessed through 9p, this isn't important, since + # everything in the store is available to the guest, + # but when building a root image it is, as all paths + # that should be available to the guest has to be + # copied to the image. + virtualisation.additionalPaths = + lib.optional + # A testScript may evaluate nodes, which has caused + # infinite recursions. The demand cycle involves: + # testScript --> + # nodes --> + # toplevel --> + # additionalPaths --> + # hasContext testScript' --> + # testScript (ad infinitum) + # If we don't need to build an image, we can break this + # cycle by short-circuiting when useNixStoreImage is false. + (config.virtualisation.useNixStoreImage && builtins.hasContext testModuleArgs.config.testScriptString && testModuleArgs.config.includeTestScriptReferences) + (hostPkgs.writeStringReferencesToFile testModuleArgs.config.testScriptString); + }; + }; +}