BoomMicrophone/nixpkgs-suyu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

integration test driver: Auto-generate integration test driver's machine

Browse source 
method documentation of nixos docs from python doc strings
This commit is contained in:
Jacek Galowicz 2023-07-07 23:01:55 +02:00
parent c916884f86
commit 846ad444c7
6 changed files with 105 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
nixos/doc/manual/default.nix
View file

@ -63,6 +63,9 @@ let
optionIdPrefix = "test-opt-";
};
testDriverMachineDocstrings = pkgs.callPackage
../../../nixos/lib/test-driver/nixos-test-driver-docstrings.nix {};
prepareManualFromMD = ''
cp -r --no-preserve=all $inputs/* .
@ -80,6 +83,8 @@ let
--replace \
'@NIXOS_TEST_OPTIONS_JSON@' \
${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json
sed -e '/@PYTHON_MACHINE_METHODS@/ {' -e 'r ${testDriverMachineDocstrings}/machine-methods.md' -e 'd' -e '}' \
-i ./development/writing-nixos-tests.section.md
'';
in rec {

205
nixos/doc/manual/development/writing-nixos-tests.section.md
View file

@ -139,210 +139,7 @@ to Python as `machine_a`.
The following methods are available on machine objects:
`start`
: Start the virtual machine. This method is asynchronous --- it does
not wait for the machine to finish booting.
`shutdown`
: Shut down the machine, waiting for the VM to exit.
`crash`
: Simulate a sudden power failure, by telling the VM to exit
immediately.
`block`
: Simulate unplugging the Ethernet cable that connects the machine to
the other machines.
`unblock`
: Undo the effect of `block`.
`screenshot`
: Take a picture of the display of the virtual machine, in PNG format.
The screenshot is linked from the HTML log.
`get_screen_text_variants`
: Return a list of different interpretations of what is currently
visible on the machine's screen using optical character
recognition. The number and order of the interpretations is not
specified and is subject to change, but if no exception is raised at
least one will be returned.
::: {.note}
This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`.
:::
`get_screen_text`
: Return a textual representation of what is currently visible on the
machine's screen using optical character recognition.
::: {.note}
This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`.
:::
`send_monitor_command`
: Send a command to the QEMU monitor. This is rarely used, but allows
doing stuff such as attaching virtual USB disks to a running
machine.
`send_key`
: Simulate pressing keys on the virtual keyboard, e.g.,
`send_key("ctrl-alt-delete")`.
`send_chars`
: Simulate typing a sequence of characters on the virtual keyboard,
e.g., `send_chars("foobar\n")` will type the string `foobar`
followed by the Enter key.
`send_console`
: Send keys to the kernel console. This allows interaction with the systemd
emergency mode, for example. Takes a string that is sent, e.g.,
`send_console("\n\nsystemctl default\n")`.
`execute`
: Execute a shell command, returning a list `(status, stdout)`.
Commands are run with `set -euo pipefail` set:
- If several commands are separated by `;` and one fails, the
command as a whole will fail.
- For pipelines, the last non-zero exit status will be returned
(if there is one; otherwise zero will be returned).
- Dereferencing unset variables fails the command.
- It will wait for stdout to be closed.
If the command detaches, it must close stdout, as `execute` will wait
for this to consume all output reliably. This can be achieved by
redirecting stdout to stderr `>&2`, to `/dev/console`, `/dev/null` or
a file. Examples of detaching commands are `sleep 365d &`, where the
shell forks a new process that can write to stdout and `xclip -i`, where
the `xclip` command itself forks without closing stdout.
Takes an optional parameter `check_return` that defaults to `True`.
Setting this parameter to `False` will not check for the return code
and return -1 instead. This can be used for commands that shut down
the VM and would therefore break the pipe that would be used for
retrieving the return code.
A timeout for the command can be specified (in seconds) using the optional
`timeout` parameter, e.g., `execute(cmd, timeout=10)` or
`execute(cmd, timeout=None)`. The default is 900 seconds.
`succeed`
: Execute a shell command, raising an exception if the exit status is
not zero, otherwise returning the standard output. Similar to `execute`,
except that the timeout is `None` by default. See `execute` for details on
command execution.
`fail`
: Like `succeed`, but raising an exception if the command returns a zero
status.
`wait_until_succeeds`
: Repeat a shell command with 1-second intervals until it succeeds.
Has a default timeout of 900 seconds which can be modified, e.g.
`wait_until_succeeds(cmd, timeout=10)`. See `execute` for details on
command execution.
`wait_until_fails`
: Like `wait_until_succeeds`, but repeating the command until it fails.
`wait_for_unit`
: Wait until the specified systemd unit has reached the "active"
state.
`wait_for_file`
: Wait until the specified file exists.
`wait_for_open_port`
: Wait until a process is listening on the given TCP port and IP address
(default `localhost`).
`wait_for_closed_port`
: Wait until nobody is listening on the given TCP port and IP address
(default `localhost`).
`wait_for_x`
: Wait until the X11 server is accepting connections.
`wait_for_text`
: Wait until the supplied regular expressions matches the textual
contents of the screen by using optical character recognition (see
`get_screen_text` and `get_screen_text_variants`).
::: {.note}
This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`.
:::
`wait_for_console_text`
: Wait until the supplied regular expressions match a line of the
serial console output. This method is useful when OCR is not
possible or accurate enough.
`wait_for_window`
: Wait until an X11 window has appeared whose name matches the given
regular expression, e.g., `wait_for_window("Terminal")`.
`copy_from_host`
: Copies a file from host to machine, e.g.,
`copy_from_host("myfile", "/etc/my/important/file")`.
The first argument is the file on the host. The file needs to be
accessible while building the nix derivation. The second argument is
the location of the file on the machine.
`systemctl`
: Runs `systemctl` commands with optional support for
`systemctl --user`
```py
machine.systemctl("list-jobs --no-pager") # runs `systemctl list-jobs --no-pager`
machine.systemctl("list-jobs --no-pager", "any-user") # spawns a shell for `any-user` and runs `systemctl --user list-jobs --no-pager`
```
`shell_interact`
: Allows you to directly interact with the guest shell. This should
only be used during test development, not in production tests.
Killing the interactive session with `Ctrl-d` or `Ctrl-c` also ends
the guest session.
`console_interact`
: Allows you to directly interact with QEMU's stdin. This should
only be used during test development, not in production tests.
Output from QEMU is only read line-wise. `Ctrl-c` kills QEMU and
`Ctrl-d` closes console and returns to the test runner.
@PYTHON_MACHINE_METHODS@
To test user units declared by `systemd.user.services` the optional
`user` argument can be used:

66
nixos/lib/test-driver/extract-docstrings.py Normal file
View file

@ -0,0 +1,66 @@
import ast
import sys
"""
This program takes all the Machine class methods and prints its methods in
markdown-style. These can then be included in the NixOS test driver
markdown style, assuming the docstrings themselves are also in markdown.
These are included in the test driver documentation in the NixOS manual.
See https://nixos.org/manual/nixos/stable/#ssec-machine-objects
The python input looks like this:
```py
...
class Machine(...):
...
def some_function(self, param1, param2):
""
documentation string of some_function.
foo bar baz.
""
...
```
Output will be:
```markdown
...
some_function(param1, param2)
: documentation string of some_function.
foo bar baz.
...
```
"""
assert len(sys.argv) == 2
with open(sys.argv[1], "r") as f:
module = ast.parse(f.read())
class_definitions = (node for node in module.body if isinstance(node, ast.ClassDef))
machine_class = next(filter(lambda x: x.name == "Machine", class_definitions))
assert machine_class is not None
function_definitions = [
node for node in machine_class.body if isinstance(node, ast.FunctionDef)
]
function_definitions.sort(key=lambda x: x.name)
for f in function_definitions:
docstr = ast.get_docstring(f)
if docstr is not None:
args = ", ".join((a.arg for a in f.args.args[1:]))
args = f"({args})"
docstr = "\n".join((f" {l}" for l in docstr.strip().splitlines()))
print(f"{f.name}{args}\n\n:{docstr[1:]}\n")

13
nixos/lib/test-driver/nixos-test-driver-docstrings.nix Normal file
View file

@ -0,0 +1,13 @@
{ runCommand
, python3
}:
let
env = { nativeBuildInputs = [ python3 ]; };
in
runCommand "nixos-test-driver-docstrings" env ''
mkdir $out
python3 ${./extract-docstrings.py} ${./test_driver/machine.py} \
> $out/machine-methods.md
''

28
nixos/lib/test-driver/test_driver/machine.py
View file

@ -417,9 +417,8 @@ class Machine:
def send_monitor_command(self, command: str) -> str:
"""
Send a command to the QEMU monitor. This is rarely used, but allows
doing stuff such as attaching virtual USB disks to a running
machine.
Send a command to the QEMU monitor. This allows attaching
virtual USB disks to a running machine, among other things.
"""
self.run_callbacks()
message = f"{command}\n".encode()
@ -630,9 +629,10 @@ class Machine:
def console_interact(self) -> None:
"""
Allows you to directly interact with QEMU's stdin.
This should only be used during test development, not in production
tests.
Allows you to directly interact with QEMU's stdin, by forwarding
terminal input to the QEMU process.
This is for use with the interactive test driver, not for production
tests, which run unattended.
Output from QEMU is only read line-wise. `Ctrl-c` kills QEMU and
`Ctrl-d` closes console and returns to the test runner.
"""
@ -885,12 +885,17 @@ class Machine:
Copies a file from host to machine, e.g.,
`copy_from_host("myfile", "/etc/my/important/file")`.
The first argument is the file on the host. The file needs to be
accessible while building the nix derivation. The second argument is
the location of the file on the machine.
The first argument is the file on the host. Note that the "host" refers
to the environment in which the test driver runs, which is typically the
Nix build sandbox.
The second argument is the location of the file on the machine that will
be written to.
The file is copied via the `shared_dir` directory which is shared among
all the VMs (using a temporary directory).
The access rights bits will mimic the ones from the host file and
user:group will be root:root.
"""
host_src = Path(source)
vm_target = Path(target)
@ -995,7 +1000,7 @@ class Machine:
"""
Wait until the supplied regular expressions match a line of the
serial console output.
This method is useful when OCR is not possible or accurate enough.
This method is useful when OCR is not possible or inaccurate.
"""
# Buffer the console output, this is needed
# to match multiline regexes.
@ -1026,6 +1031,9 @@ class Machine:
"""
Simulate pressing keys on the virtual keyboard, e.g.,
`send_key("ctrl-alt-delete")`.
Please also refer to the QEMU documentation for more information on the
input syntax: https://en.wikibooks.org/wiki/QEMU/Monitor#sendkey_keys
"""
key = CHAR_TO_KEY.get(key, key)
context = self.nested(f"sending key {repr(key)}") if log else nullcontext()

4
nixos/lib/testing/driver.nix
View file

@ -66,7 +66,7 @@ let
echo -n "$testScript" >> testScriptWithTypes
echo "Running type check (enable/disable: config.skipTypeCheck)"
echo "See https://nixos.org/manual/nixos/stable/#sec-test-options-reference"
echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck"
mypy --no-implicit-optional \
--pretty \
@ -81,7 +81,7 @@ let
${testDriver}/bin/generate-driver-symbols
${lib.optionalString (!config.skipLint) ''
echo "Linting test script (enable/disable: config.skipLint)"
echo "See https://nixos.org/manual/nixos/stable/#sec-test-options-reference"
echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint"
PYFLAKES_BUILTINS="$(
echo -n ${lib.escapeShellArg (lib.concatStringsSep "," pythonizedNames)},