From 846ad444c722abf49d744814fde831cd3c21d599 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Fri, 7 Jul 2023 23:01:55 +0200 Subject: [PATCH] integration test driver: Auto-generate integration test driver's machine method documentation of nixos docs from python doc strings --- nixos/doc/manual/default.nix | 5 + .../writing-nixos-tests.section.md | 205 +----------------- nixos/lib/test-driver/extract-docstrings.py | 66 ++++++ .../nixos-test-driver-docstrings.nix | 13 ++ nixos/lib/test-driver/test_driver/machine.py | 28 ++- nixos/lib/testing/driver.nix | 4 +- 6 files changed, 105 insertions(+), 216 deletions(-) create mode 100644 nixos/lib/test-driver/extract-docstrings.py create mode 100644 nixos/lib/test-driver/nixos-test-driver-docstrings.nix diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 40af4c1fa0b0..f2fd6a682934 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -63,6 +63,9 @@ let optionIdPrefix = "test-opt-"; }; + testDriverMachineDocstrings = pkgs.callPackage + ../../../nixos/lib/test-driver/nixos-test-driver-docstrings.nix {}; + prepareManualFromMD = '' cp -r --no-preserve=all $inputs/* . @@ -80,6 +83,8 @@ let --replace \ '@NIXOS_TEST_OPTIONS_JSON@' \ ${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json + sed -e '/@PYTHON_MACHINE_METHODS@/ {' -e 'r ${testDriverMachineDocstrings}/machine-methods.md' -e 'd' -e '}' \ + -i ./development/writing-nixos-tests.section.md ''; in rec { diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md index 486a4b64a262..84b247fd2042 100644 --- a/nixos/doc/manual/development/writing-nixos-tests.section.md +++ b/nixos/doc/manual/development/writing-nixos-tests.section.md @@ -139,210 +139,7 @@ to Python as `machine_a`. The following methods are available on machine objects: -`start` - -: Start the virtual machine. This method is asynchronous --- it does - not wait for the machine to finish booting. - -`shutdown` - -: Shut down the machine, waiting for the VM to exit. - -`crash` - -: Simulate a sudden power failure, by telling the VM to exit - immediately. - -`block` - -: Simulate unplugging the Ethernet cable that connects the machine to - the other machines. - -`unblock` - -: Undo the effect of `block`. - -`screenshot` - -: Take a picture of the display of the virtual machine, in PNG format. - The screenshot is linked from the HTML log. - -`get_screen_text_variants` - -: Return a list of different interpretations of what is currently - visible on the machine's screen using optical character - recognition. The number and order of the interpretations is not - specified and is subject to change, but if no exception is raised at - least one will be returned. - - ::: {.note} - This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`. - ::: - -`get_screen_text` - -: Return a textual representation of what is currently visible on the - machine's screen using optical character recognition. - - ::: {.note} - This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`. - ::: - -`send_monitor_command` - -: Send a command to the QEMU monitor. This is rarely used, but allows - doing stuff such as attaching virtual USB disks to a running - machine. - -`send_key` - -: Simulate pressing keys on the virtual keyboard, e.g., - `send_key("ctrl-alt-delete")`. - -`send_chars` - -: Simulate typing a sequence of characters on the virtual keyboard, - e.g., `send_chars("foobar\n")` will type the string `foobar` - followed by the Enter key. - -`send_console` - -: Send keys to the kernel console. This allows interaction with the systemd - emergency mode, for example. Takes a string that is sent, e.g., - `send_console("\n\nsystemctl default\n")`. - -`execute` - -: Execute a shell command, returning a list `(status, stdout)`. - - Commands are run with `set -euo pipefail` set: - - - If several commands are separated by `;` and one fails, the - command as a whole will fail. - - - For pipelines, the last non-zero exit status will be returned - (if there is one; otherwise zero will be returned). - - - Dereferencing unset variables fails the command. - - - It will wait for stdout to be closed. - - If the command detaches, it must close stdout, as `execute` will wait - for this to consume all output reliably. This can be achieved by - redirecting stdout to stderr `>&2`, to `/dev/console`, `/dev/null` or - a file. Examples of detaching commands are `sleep 365d &`, where the - shell forks a new process that can write to stdout and `xclip -i`, where - the `xclip` command itself forks without closing stdout. - - Takes an optional parameter `check_return` that defaults to `True`. - Setting this parameter to `False` will not check for the return code - and return -1 instead. This can be used for commands that shut down - the VM and would therefore break the pipe that would be used for - retrieving the return code. - - A timeout for the command can be specified (in seconds) using the optional - `timeout` parameter, e.g., `execute(cmd, timeout=10)` or - `execute(cmd, timeout=None)`. The default is 900 seconds. - -`succeed` - -: Execute a shell command, raising an exception if the exit status is - not zero, otherwise returning the standard output. Similar to `execute`, - except that the timeout is `None` by default. See `execute` for details on - command execution. - -`fail` - -: Like `succeed`, but raising an exception if the command returns a zero - status. - -`wait_until_succeeds` - -: Repeat a shell command with 1-second intervals until it succeeds. - Has a default timeout of 900 seconds which can be modified, e.g. - `wait_until_succeeds(cmd, timeout=10)`. See `execute` for details on - command execution. - -`wait_until_fails` - -: Like `wait_until_succeeds`, but repeating the command until it fails. - -`wait_for_unit` - -: Wait until the specified systemd unit has reached the "active" - state. - -`wait_for_file` - -: Wait until the specified file exists. - -`wait_for_open_port` - -: Wait until a process is listening on the given TCP port and IP address - (default `localhost`). - -`wait_for_closed_port` - -: Wait until nobody is listening on the given TCP port and IP address - (default `localhost`). - -`wait_for_x` - -: Wait until the X11 server is accepting connections. - -`wait_for_text` - -: Wait until the supplied regular expressions matches the textual - contents of the screen by using optical character recognition (see - `get_screen_text` and `get_screen_text_variants`). - - ::: {.note} - This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`. - ::: - -`wait_for_console_text` - -: Wait until the supplied regular expressions match a line of the - serial console output. This method is useful when OCR is not - possible or accurate enough. - -`wait_for_window` - -: Wait until an X11 window has appeared whose name matches the given - regular expression, e.g., `wait_for_window("Terminal")`. - -`copy_from_host` - -: Copies a file from host to machine, e.g., - `copy_from_host("myfile", "/etc/my/important/file")`. - - The first argument is the file on the host. The file needs to be - accessible while building the nix derivation. The second argument is - the location of the file on the machine. - -`systemctl` - -: Runs `systemctl` commands with optional support for - `systemctl --user` - - ```py - machine.systemctl("list-jobs --no-pager") # runs `systemctl list-jobs --no-pager` - machine.systemctl("list-jobs --no-pager", "any-user") # spawns a shell for `any-user` and runs `systemctl --user list-jobs --no-pager` - ``` - -`shell_interact` - -: Allows you to directly interact with the guest shell. This should - only be used during test development, not in production tests. - Killing the interactive session with `Ctrl-d` or `Ctrl-c` also ends - the guest session. - -`console_interact` - -: Allows you to directly interact with QEMU's stdin. This should - only be used during test development, not in production tests. - Output from QEMU is only read line-wise. `Ctrl-c` kills QEMU and - `Ctrl-d` closes console and returns to the test runner. +@PYTHON_MACHINE_METHODS@ To test user units declared by `systemd.user.services` the optional `user` argument can be used: diff --git a/nixos/lib/test-driver/extract-docstrings.py b/nixos/lib/test-driver/extract-docstrings.py new file mode 100644 index 000000000000..5aec4c89a9d7 --- /dev/null +++ b/nixos/lib/test-driver/extract-docstrings.py @@ -0,0 +1,66 @@ +import ast +import sys + +""" +This program takes all the Machine class methods and prints its methods in +markdown-style. These can then be included in the NixOS test driver +markdown style, assuming the docstrings themselves are also in markdown. + +These are included in the test driver documentation in the NixOS manual. +See https://nixos.org/manual/nixos/stable/#ssec-machine-objects + +The python input looks like this: + +```py +... + +class Machine(...): + ... + + def some_function(self, param1, param2): + "" + documentation string of some_function. + foo bar baz. + "" + ... +``` + +Output will be: + +```markdown +... + +some_function(param1, param2) + +: documentation string of some_function. + foo bar baz. + +... +``` + +""" + +assert len(sys.argv) == 2 + +with open(sys.argv[1], "r") as f: + module = ast.parse(f.read()) + +class_definitions = (node for node in module.body if isinstance(node, ast.ClassDef)) + +machine_class = next(filter(lambda x: x.name == "Machine", class_definitions)) +assert machine_class is not None + +function_definitions = [ + node for node in machine_class.body if isinstance(node, ast.FunctionDef) +] +function_definitions.sort(key=lambda x: x.name) + +for f in function_definitions: + docstr = ast.get_docstring(f) + if docstr is not None: + args = ", ".join((a.arg for a in f.args.args[1:])) + args = f"({args})" + + docstr = "\n".join((f" {l}" for l in docstr.strip().splitlines())) + + print(f"{f.name}{args}\n\n:{docstr[1:]}\n") diff --git a/nixos/lib/test-driver/nixos-test-driver-docstrings.nix b/nixos/lib/test-driver/nixos-test-driver-docstrings.nix new file mode 100644 index 000000000000..a3ef50e4e820 --- /dev/null +++ b/nixos/lib/test-driver/nixos-test-driver-docstrings.nix @@ -0,0 +1,13 @@ +{ runCommand +, python3 +}: + +let + env = { nativeBuildInputs = [ python3 ]; }; +in + +runCommand "nixos-test-driver-docstrings" env '' + mkdir $out + python3 ${./extract-docstrings.py} ${./test_driver/machine.py} \ + > $out/machine-methods.md +'' diff --git a/nixos/lib/test-driver/test_driver/machine.py b/nixos/lib/test-driver/test_driver/machine.py index 8a5eebdc820b..dbfe256011ee 100644 --- a/nixos/lib/test-driver/test_driver/machine.py +++ b/nixos/lib/test-driver/test_driver/machine.py @@ -417,9 +417,8 @@ class Machine: def send_monitor_command(self, command: str) -> str: """ - Send a command to the QEMU monitor. This is rarely used, but allows - doing stuff such as attaching virtual USB disks to a running - machine. + Send a command to the QEMU monitor. This allows attaching + virtual USB disks to a running machine, among other things. """ self.run_callbacks() message = f"{command}\n".encode() @@ -630,9 +629,10 @@ class Machine: def console_interact(self) -> None: """ - Allows you to directly interact with QEMU's stdin. - This should only be used during test development, not in production - tests. + Allows you to directly interact with QEMU's stdin, by forwarding + terminal input to the QEMU process. + This is for use with the interactive test driver, not for production + tests, which run unattended. Output from QEMU is only read line-wise. `Ctrl-c` kills QEMU and `Ctrl-d` closes console and returns to the test runner. """ @@ -885,12 +885,17 @@ class Machine: Copies a file from host to machine, e.g., `copy_from_host("myfile", "/etc/my/important/file")`. - The first argument is the file on the host. The file needs to be - accessible while building the nix derivation. The second argument is - the location of the file on the machine. + The first argument is the file on the host. Note that the "host" refers + to the environment in which the test driver runs, which is typically the + Nix build sandbox. + + The second argument is the location of the file on the machine that will + be written to. The file is copied via the `shared_dir` directory which is shared among all the VMs (using a temporary directory). + The access rights bits will mimic the ones from the host file and + user:group will be root:root. """ host_src = Path(source) vm_target = Path(target) @@ -995,7 +1000,7 @@ class Machine: """ Wait until the supplied regular expressions match a line of the serial console output. - This method is useful when OCR is not possible or accurate enough. + This method is useful when OCR is not possible or inaccurate. """ # Buffer the console output, this is needed # to match multiline regexes. @@ -1026,6 +1031,9 @@ class Machine: """ Simulate pressing keys on the virtual keyboard, e.g., `send_key("ctrl-alt-delete")`. + + Please also refer to the QEMU documentation for more information on the + input syntax: https://en.wikibooks.org/wiki/QEMU/Monitor#sendkey_keys """ key = CHAR_TO_KEY.get(key, key) context = self.nested(f"sending key {repr(key)}") if log else nullcontext() diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix index 5eb62d0b32c8..23574698c062 100644 --- a/nixos/lib/testing/driver.nix +++ b/nixos/lib/testing/driver.nix @@ -66,7 +66,7 @@ let echo -n "$testScript" >> testScriptWithTypes echo "Running type check (enable/disable: config.skipTypeCheck)" - echo "See https://nixos.org/manual/nixos/stable/#sec-test-options-reference" + echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck" mypy --no-implicit-optional \ --pretty \ @@ -81,7 +81,7 @@ let ${testDriver}/bin/generate-driver-symbols ${lib.optionalString (!config.skipLint) '' echo "Linting test script (enable/disable: config.skipLint)" - echo "See https://nixos.org/manual/nixos/stable/#sec-test-options-reference" + echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint" PYFLAKES_BUILTINS="$( echo -n ${lib.escapeShellArg (lib.concatStringsSep "," pythonizedNames)},