Merge pull request #6093 from wernerlewis/bignum_test_script

Add bignum test case generation script

scripts/make_generated_files.bat

@@ -10,4 +10,5 @@ perl scripts\generate_features.pl || exit /b 1
 python scripts\generate_ssl_debug_helpers.py || exit /b 1
 perl scripts\generate_visualc_files.pl || exit /b 1
 python scripts\generate_psa_constants.py || exit /b 1
+python tests\scripts\generate_bignum_tests.py || exit /b 1
 python tests\scripts\generate_psa_tests.py || exit /b 1

scripts/mbedtls_dev/test_generation.py

@@ -0,0 +1,219 @@
+"""Common test generation classes and main function.
+
+These are used both by generate_psa_tests.py and generate_bignum_tests.py.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import posixpath
+import re
+
+from abc import ABCMeta, abstractmethod
+from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar
+
+from mbedtls_dev import build_tree
+from mbedtls_dev import test_case
+
+T = TypeVar('T') #pylint: disable=invalid-name
+
+
+class BaseTarget(metaclass=ABCMeta):
+    """Base target for test case generation.
+
+    Child classes of this class represent an output file, and can be referred
+    to as file targets. These indicate where test cases will be written to for
+    all subclasses of the file target, which is set by `target_basename`.
+
+    Attributes:
+        count: Counter for test cases from this class.
+        case_description: Short description of the test case. This may be
+            automatically generated using the class, or manually set.
+        dependencies: A list of dependencies required for the test case.
+        show_test_count: Toggle for inclusion of `count` in the test description.
+        target_basename: Basename of file to write generated tests to. This
+            should be specified in a child class of BaseTarget.
+        test_function: Test function which the class generates cases for.
+        test_name: A common name or description of the test function. This can
+            be `test_function`, a clearer equivalent, or a short summary of the
+            test function's purpose.
+    """
+    count = 0
+    case_description = ""
+    dependencies = [] # type: List[str]
+    show_test_count = True
+    target_basename = ""
+    test_function = ""
+    test_name = ""
+
+    def __new__(cls, *args, **kwargs):
+        # pylint: disable=unused-argument
+        cls.count += 1
+        return super().__new__(cls)
+
+    @abstractmethod
+    def arguments(self) -> List[str]:
+        """Get the list of arguments for the test case.
+
+        Override this method to provide the list of arguments required for
+        the `test_function`.
+
+        Returns:
+            List of arguments required for the test function.
+        """
+        raise NotImplementedError
+
+    def description(self) -> str:
+        """Create a test case description.
+
+        Creates a description of the test case, including a name for the test
+        function, an optional case count, and a description of the specific
+        test case. This should inform a reader what is being tested, and
+        provide context for the test case.
+
+        Returns:
+            Description for the test case.
+        """
+        if self.show_test_count:
+            return "{} #{} {}".format(
+                self.test_name, self.count, self.case_description
+                ).strip()
+        else:
+            return "{} {}".format(self.test_name, self.case_description).strip()
+
+
+    def create_test_case(self) -> test_case.TestCase:
+        """Generate TestCase from the instance."""
+        tc = test_case.TestCase()
+        tc.set_description(self.description())
+        tc.set_function(self.test_function)
+        tc.set_arguments(self.arguments())
+        tc.set_dependencies(self.dependencies)
+
+        return tc
+
+    @classmethod
+    @abstractmethod
+    def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
+        """Generate test cases for the class test function.
+
+        This will be called in classes where `test_function` is set.
+        Implementations should yield TestCase objects, by creating instances
+        of the class with appropriate input data, and then calling
+        `create_test_case()` on each.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def generate_tests(cls) -> Iterator[test_case.TestCase]:
+        """Generate test cases for the class and its subclasses.
+
+        In classes with `test_function` set, `generate_function_tests()` is
+        called to generate test cases first.
+
+        In all classes, this method will iterate over its subclasses, and
+        yield from `generate_tests()` in each. Calling this method on a class X
+        will yield test cases from all classes derived from X.
+        """
+        if cls.test_function:
+            yield from cls.generate_function_tests()
+        for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__):
+            yield from subclass.generate_tests()
+
+
+class TestGenerator:
+    """Generate test cases and write to data files."""
+    def __init__(self, options) -> None:
+        self.test_suite_directory = self.get_option(options, 'directory',
+                                                    'tests/suites')
+        # Update `targets` with an entry for each child class of BaseTarget.
+        # Each entry represents a file generated by the BaseTarget framework,
+        # and enables generating the .data files using the CLI.
+        self.targets.update({
+            subclass.target_basename: subclass.generate_tests
+            for subclass in BaseTarget.__subclasses__()
+        })
+
+    @staticmethod
+    def get_option(options, name: str, default: T) -> T:
+        value = getattr(options, name, None)
+        return default if value is None else value
+
+    def filename_for(self, basename: str) -> str:
+        """The location of the data file with the specified base name."""
+        return posixpath.join(self.test_suite_directory, basename + '.data')
+
+    def write_test_data_file(self, basename: str,
+                             test_cases: Iterable[test_case.TestCase]) -> None:
+        """Write the test cases to a .data file.
+
+        The output file is ``basename + '.data'`` in the test suite directory.
+        """
+        filename = self.filename_for(basename)
+        test_case.write_data_file(filename, test_cases)
+
+    # Note that targets whose names contain 'test_format' have their content
+    # validated by `abi_check.py`.
+    targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
+
+    def generate_target(self, name: str, *target_args) -> None:
+        """Generate cases and write to data file for a target.
+
+        For target callables which require arguments, override this function
+        and pass these arguments using super() (see PSATestGenerator).
+        """
+        test_cases = self.targets[name](*target_args)
+        self.write_test_data_file(name, test_cases)
+
+def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
+    """Command line entry point."""
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('--list', action='store_true',
+                        help='List available targets and exit')
+    parser.add_argument('--list-for-cmake', action='store_true',
+                        help='Print \';\'-separated list of available targets and exit')
+    parser.add_argument('--directory', metavar='DIR',
+                        help='Output directory (default: tests/suites)')
+    # The `--directory` option is interpreted relative to the directory from
+    # which the script is invoked, but the default is relative to the root of
+    # the mbedtls tree. The default should not be set above, but instead after
+    # `build_tree.chdir_to_root()` is called.
+    parser.add_argument('targets', nargs='*', metavar='TARGET',
+                        help='Target file to generate (default: all; "-": none)')
+    options = parser.parse_args(args)
+    build_tree.chdir_to_root()
+    generator = generator_class(options)
+    if options.list:
+        for name in sorted(generator.targets):
+            print(generator.filename_for(name))
+        return
+    # List in a cmake list format (i.e. ';'-separated)
+    if options.list_for_cmake:
+        print(';'.join(generator.filename_for(name)
+                       for name in sorted(generator.targets)), end='')
+        return
+    if options.targets:
+        # Allow "-" as a special case so you can run
+        # ``generate_xxx_tests.py - $targets`` and it works uniformly whether
+        # ``$targets`` is empty or not.
+        options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
+                           for target in options.targets
+                           if target != '-']
+    else:
+        options.targets = sorted(generator.targets)
+    for target in options.targets:
+        generator.generate_target(target)

tests/CMakeLists.txt

@@ -17,6 +17,17 @@ endif()
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/suites)
 
 # Get base names for generated files (starting at "suites/")
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
+        --list-for-cmake
+        --directory suites
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_bignum_generated_data_files)
+
 execute_process(
     COMMAND
         ${MBEDTLS_PYTHON_EXECUTABLE}
@@ -26,18 +37,37 @@ execute_process(
     WORKING_DIRECTORY
         ${CMAKE_CURRENT_SOURCE_DIR}/..
     OUTPUT_VARIABLE
-        base_generated_data_files)
+        base_psa_generated_data_files)
 
 # Derive generated file paths in the build directory
-set(generated_data_files "")
+set(base_generated_data_files ${base_bignum_generated_data_files} ${base_psa_generated_data_files})
-foreach(file ${base_generated_data_files})
+set(bignum_generated_data_files "")
-    list(APPEND generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
+set(psa_generated_data_files "")
+foreach(file ${base_bignum_generated_data_files})
+    list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
+endforeach()
+foreach(file ${base_psa_generated_data_files})
+    list(APPEND psa_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
 endforeach()
 
 if(GEN_FILES)
     add_custom_command(
         OUTPUT
-            ${generated_data_files}
+            ${bignum_generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py
+    )
+    add_custom_command(
+        OUTPUT
+            ${psa_generated_data_files}
         WORKING_DIRECTORY
             ${CMAKE_CURRENT_SOURCE_DIR}/..
         COMMAND
@@ -50,6 +80,7 @@ if(GEN_FILES)
             ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/macro_collector.py
             ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/psa_storage.py
             ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py
             ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_config.h
             ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_values.h
             ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_extra.h
@@ -65,7 +96,8 @@ endif()
 # they can cause race conditions in parallel builds.
 # With this line, only 4 sub-makefiles include the above command, that reduces
 # the risk of a race.
-add_custom_target(test_suite_generated_data DEPENDS ${generated_data_files})
+add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files})
+add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files})
 # Test suites caught by SKIP_TEST_SUITES are built but not executed.
 # "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
 # but not "test_suite_foobar".
@@ -82,23 +114,39 @@ function(add_test_suite suite_name)
 
     # Get the test names of the tests with generated .data files
     # from the generated_data_files list in parent scope.
-    set(generated_data_names "")
+    set(bignum_generated_data_names "")
-    foreach(generated_data_file ${generated_data_files})
+    set(psa_generated_data_names "")
+    foreach(generated_data_file ${bignum_generated_data_files})
         # Get the plain filename
         get_filename_component(generated_data_name ${generated_data_file} NAME)
         # Remove the ".data" extension
         get_name_without_last_ext(generated_data_name ${generated_data_name})
         # Remove leading "test_suite_"
         string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
-        list(APPEND generated_data_names ${generated_data_name})
+        list(APPEND bignum_generated_data_names ${generated_data_name})
+    endforeach()
+    foreach(generated_data_file ${psa_generated_data_files})
+        # Get the plain filename
+        get_filename_component(generated_data_name ${generated_data_file} NAME)
+        # Remove the ".data" extension
+        get_name_without_last_ext(generated_data_name ${generated_data_name})
+        # Remove leading "test_suite_"
+        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
+        list(APPEND psa_generated_data_names ${generated_data_name})
     endforeach()
 
-    if(";${generated_data_names};" MATCHES ";${data_name};")
+    if(";${bignum_generated_data_names};" MATCHES ";${data_name};")
         set(data_file
             ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_bignum_generated_data)
+    elseif(";${psa_generated_data_names};" MATCHES ";${data_name};")
+        set(data_file
+            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_psa_generated_data)
     else()
         set(data_file
             ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_bignum_generated_data test_suite_psa_generated_data)
     endif()
 
     add_custom_command(
@@ -129,7 +177,7 @@ function(add_test_suite suite_name)
     )
 
     add_executable(test_suite_${data_name} test_suite_${data_name}.c $<TARGET_OBJECTS:mbedtls_test>)
-    add_dependencies(test_suite_${data_name} test_suite_generated_data)
+    add_dependencies(test_suite_${data_name} ${dependency})
     target_link_libraries(test_suite_${data_name} ${libs})
     # Include test-specific header files from ./include and private header
     # files (used by some invasive tests) from ../library. Public header

tests/Makefile

@@ -66,28 +66,45 @@ PYTHON ?= $(shell if type python3 >/dev/null 2>/dev/null; then echo python3; els
 endif
 
 .PHONY: generated_files
-GENERATED_DATA_FILES := $(patsubst tests/%,%,$(shell \
+GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \
+	$(PYTHON) scripts/generate_bignum_tests.py --list || \
+	echo FAILED \
+))
+ifeq ($(GENERATED_BIGNUM_DATA_FILES),FAILED)
+$(error "$(PYTHON) scripts/generate_bignum_tests.py --list" failed)
+endif
+GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \
 	$(PYTHON) scripts/generate_psa_tests.py --list || \
 	echo FAILED \
 ))
-ifeq ($(GENERATED_DATA_FILES),FAILED)
+ifeq ($(GENERATED_PSA_DATA_FILES),FAILED)
 $(error "$(PYTHON) scripts/generate_psa_tests.py --list" failed)
 endif
-GENERATED_FILES := $(GENERATED_DATA_FILES)
+GENERATED_FILES := $(GENERATED_PSA_DATA_FILES) $(GENERATED_BIGNUM_DATA_FILES)
 generated_files: $(GENERATED_FILES)
 
-# generate_psa_tests.py spends more time analyzing inputs than generating
+# generate_bignum_tests.py and generate_psa_tests.py spend more time analyzing
-# outputs. Its inputs are the same no matter which files are being generated.
+# inputs than generating outputs. Its inputs are the same no matter which files
+# are being generated.
 # It's rare not to want all the outputs. So always generate all of its outputs.
 # Use an intermediate phony dependency so that parallel builds don't run
 # a separate instance of the recipe for each output file.
-.SECONDARY: generated_psa_test_data
+.SECONDARY: generated_bignum_test_data generated_psa_test_data
-$(GENERATED_DATA_FILES): generated_psa_test_data
+$(GENERATED_BIGNUM_DATA_FILES): generated_bignum_test_data
+generated_bignum_test_data: scripts/generate_bignum_tests.py
+generated_bignum_test_data: ../scripts/mbedtls_dev/test_case.py
+generated_bignum_test_data: ../scripts/mbedtls_dev/test_generation.py
+generated_bignum_test_data:
+	echo "  Gen   $(GENERATED_BIGNUM_DATA_FILES)"
+	$(PYTHON) scripts/generate_bignum_tests.py
+
+$(GENERATED_PSA_DATA_FILES): generated_psa_test_data
 generated_psa_test_data: scripts/generate_psa_tests.py
 generated_psa_test_data: ../scripts/mbedtls_dev/crypto_knowledge.py
 generated_psa_test_data: ../scripts/mbedtls_dev/macro_collector.py
 generated_psa_test_data: ../scripts/mbedtls_dev/psa_storage.py
 generated_psa_test_data: ../scripts/mbedtls_dev/test_case.py
+generated_psa_test_data: ../scripts/mbedtls_dev/test_generation.py
 ## The generated file only depends on the options that are present in
 ## crypto_config.h, not on which options are set. To avoid regenerating this
 ## file all the time when switching between configurations, don't declare
@@ -98,7 +115,7 @@ generated_psa_test_data: ../include/psa/crypto_values.h
 generated_psa_test_data: ../include/psa/crypto_extra.h
 generated_psa_test_data: suites/test_suite_psa_crypto_metadata.data
 generated_psa_test_data:
-	echo "  Gen   $(GENERATED_DATA_FILES) ..."
+	echo "  Gen   $(GENERATED_PSA_DATA_FILES) ..."
 	$(PYTHON) scripts/generate_psa_tests.py
 
 # A test application is built for each suites/test_suite_*.data file.
@@ -107,7 +124,7 @@ generated_psa_test_data:
 DATA_FILES := $(wildcard suites/test_suite_*.data)
 # Make sure that generated data files are included even if they don't
 # exist yet when the makefile is parsed.
-DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_DATA_FILES))
+DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_FILES))
 APPS = $(basename $(subst suites/,,$(DATA_FILES)))
 
 # Construct executable name by adding OS specific suffix $(EXEXT).

tests/scripts/check-generated-files.sh

@@ -126,4 +126,5 @@ check scripts/generate_ssl_debug_helpers.py library/ssl_debug_helpers_generated.
 # the step that creates or updates these files.
 check scripts/generate_visualc_files.pl visualc/VS2010
 check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c
+check tests/scripts/generate_bignum_tests.py $(tests/scripts/generate_bignum_tests.py --list)
 check tests/scripts/generate_psa_tests.py $(tests/scripts/generate_psa_tests.py --list)

tests/scripts/generate_bignum_tests.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""Generate test data for bignum functions.
+
+With no arguments, generate all test data. With non-option arguments,
+generate only the specified files.
+
+Class structure:
+
+Child classes of test_generation.BaseTarget (file targets) represent an output
+file. These indicate where test cases will be written to, for all subclasses of
+this target. Multiple file targets should not reuse a `target_basename`.
+
+Each subclass derived from a file target can either be:
+  - A concrete class, representing a test function, which generates test cases.
+  - An abstract class containing shared methods and attributes, not associated
+        with a test function. An example is BignumOperation, which provides
+        common features used for bignum binary operations.
+
+Both concrete and abstract subclasses can be derived from, to implement
+additional test cases (see BignumCmp and BignumCmpAbs for examples of deriving
+from abstract and concrete classes).
+
+
+Adding test case generation for a function:
+
+A subclass representing the test function should be added, deriving from a
+file target such as BignumTarget. This test class must set/implement the
+following:
+  - test_function: the function name from the associated .function file.
+  - test_name: a descriptive name or brief summary to refer to the test
+        function.
+  - arguments(): a method to generate the list of arguments required for the
+        test_function.
+  - generate_function_test(): a method to generate TestCases for the function.
+        This should create instances of the class with required input data, and
+        call `.create_test_case()` to yield the TestCase.
+
+Additional details and other attributes/methods are given in the documentation
+of BaseTarget in test_generation.py.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import sys
+import typing
+
+from abc import ABCMeta, abstractmethod
+from typing import Iterator, List, Tuple, TypeVar
+
+import scripts_path # pylint: disable=unused-import
+from mbedtls_dev import test_case
+from mbedtls_dev import test_generation
+
+T = TypeVar('T') #pylint: disable=invalid-name
+
+def hex_to_int(val: str) -> int:
+    return int(val, 16) if val else 0
+
+def quote_str(val) -> str:
+    return "\"{}\"".format(val)
+
+def combination_pairs(values: List[T]) -> List[Tuple[T, T]]:
+    """Return all pair combinations from input values.
+
+    The return value is cast, as older versions of mypy are unable to derive
+    the specific type returned by itertools.combinations_with_replacement.
+    """
+    return typing.cast(
+        List[Tuple[T, T]],
+        list(itertools.combinations_with_replacement(values, 2))
+    )
+
+
+class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta):
+    #pylint: disable=abstract-method
+    """Target for bignum (mpi) test case generation."""
+    target_basename = 'test_suite_mpi.generated'
+
+
+class BignumOperation(BignumTarget, metaclass=ABCMeta):
+    """Common features for bignum binary operations.
+
+    This adds functionality common in binary operation tests. This includes
+    generation of case descriptions, using descriptions of values and symbols
+    to represent the operation or result.
+
+    Attributes:
+        symbol: Symbol used for the operation in case description.
+        input_values: List of values to use as test case inputs. These are
+            combined to produce pairs of values.
+        input_cases: List of tuples containing pairs of test case inputs. This
+            can be used to implement specific pairs of inputs.
+    """
+    symbol = ""
+    input_values = [
+        "", "0", "7b", "-7b",
+        "0000000000000000123", "-0000000000000000123",
+        "1230000000000000000", "-1230000000000000000"
+    ] # type: List[str]
+    input_cases = [] # type: List[Tuple[str, str]]
+
+    def __init__(self, val_a: str, val_b: str) -> None:
+        self.arg_a = val_a
+        self.arg_b = val_b
+        self.int_a = hex_to_int(val_a)
+        self.int_b = hex_to_int(val_b)
+
+    def arguments(self) -> List[str]:
+        return [quote_str(self.arg_a), quote_str(self.arg_b), self.result()]
+
+    def description(self) -> str:
+        """Generate a description for the test case.
+
+        If not set, case_description uses the form A `symbol` B, where symbol
+        is used to represent the operation. Descriptions of each value are
+        generated to provide some context to the test case.
+        """
+        if not self.case_description:
+            self.case_description = "{} {} {}".format(
+                self.value_description(self.arg_a),
+                self.symbol,
+                self.value_description(self.arg_b)
+            )
+        return super().description()
+
+    @abstractmethod
+    def result(self) -> str:
+        """Get the result of the operation.
+
+        This could be calculated during initialization and stored as `_result`
+        and then returned, or calculated when the method is called.
+        """
+        raise NotImplementedError
+
+    @staticmethod
+    def value_description(val) -> str:
+        """Generate a description of the argument val.
+
+        This produces a simple description of the value, which is used in test
+        case naming to add context.
+        """
+        if val == "":
+            return "0 (null)"
+        if val == "0":
+            return "0 (1 limb)"
+
+        if val[0] == "-":
+            tmp = "negative"
+            val = val[1:]
+        else:
+            tmp = "positive"
+        if val[0] == "0":
+            tmp += " with leading zero limb"
+        elif len(val) > 10:
+            tmp = "large " + tmp
+        return tmp
+
+    @classmethod
+    def get_value_pairs(cls) -> Iterator[Tuple[str, str]]:
+        """Generator to yield pairs of inputs.
+
+        Combinations are first generated from all input values, and then
+        specific cases provided.
+        """
+        yield from combination_pairs(cls.input_values)
+        yield from cls.input_cases
+
+    @classmethod
+    def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
+        for a_value, b_value in cls.get_value_pairs():
+            cur_op = cls(a_value, b_value)
+            yield cur_op.create_test_case()
+
+
+class BignumCmp(BignumOperation):
+    """Test cases for bignum value comparison."""
+    count = 0
+    test_function = "mbedtls_mpi_cmp_mpi"
+    test_name = "MPI compare"
+    input_cases = [
+        ("-2", "-3"),
+        ("-2", "-2"),
+        ("2b4", "2b5"),
+        ("2b5", "2b6")
+        ]
+
+    def __init__(self, val_a, val_b) -> None:
+        super().__init__(val_a, val_b)
+        self._result = int(self.int_a > self.int_b) - int(self.int_a < self.int_b)
+        self.symbol = ["<", "==", ">"][self._result + 1]
+
+    def result(self) -> str:
+        return str(self._result)
+
+
+class BignumCmpAbs(BignumCmp):
+    """Test cases for absolute bignum value comparison."""
+    count = 0
+    test_function = "mbedtls_mpi_cmp_abs"
+    test_name = "MPI compare (abs)"
+
+    def __init__(self, val_a, val_b) -> None:
+        super().__init__(val_a.strip("-"), val_b.strip("-"))
+
+
+class BignumAdd(BignumOperation):
+    """Test cases for bignum value addition."""
+    count = 0
+    symbol = "+"
+    test_function = "mbedtls_mpi_add_mpi"
+    test_name = "MPI add"
+    input_cases = combination_pairs(
+        [
+            "1c67967269c6", "9cde3",
+            "-1c67967269c6", "-9cde3",
+        ]
+    )
+
+    def result(self) -> str:
+        return quote_str("{:x}".format(self.int_a + self.int_b))
+
+if __name__ == '__main__':
+    # Use the section of the docstring relevant to the CLI as description
+    test_generation.main(sys.argv[1:], "\n".join(__doc__.splitlines()[:4]))

tests/scripts/generate_psa_tests.py

@@ -20,22 +20,17 @@ generate only the specified files.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import argparse
 import enum
-import os
-import posixpath
 import re
 import sys
-from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional, TypeVar
+from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional
 
 import scripts_path # pylint: disable=unused-import
-from mbedtls_dev import build_tree
 from mbedtls_dev import crypto_knowledge
 from mbedtls_dev import macro_collector
 from mbedtls_dev import psa_storage
 from mbedtls_dev import test_case
-
+from mbedtls_dev import test_generation
-T = TypeVar('T') #pylint: disable=invalid-name
 
 
 def psa_want_symbol(name: str) -> str:
@@ -897,35 +892,11 @@ class StorageFormatV0(StorageFormat):
         yield from super().generate_all_keys()
         yield from self.all_keys_for_implicit_usage()
 
-class TestGenerator:
+class PSATestGenerator(test_generation.TestGenerator):
-    """Generate test data."""
+    """Test generator subclass including PSA targets and info."""
-
-    def __init__(self, options) -> None:
-        self.test_suite_directory = self.get_option(options, 'directory',
-                                                    'tests/suites')
-        self.info = Information()
-
-    @staticmethod
-    def get_option(options, name: str, default: T) -> T:
-        value = getattr(options, name, None)
-        return default if value is None else value
-
-    def filename_for(self, basename: str) -> str:
-        """The location of the data file with the specified base name."""
-        return posixpath.join(self.test_suite_directory, basename + '.data')
-
-    def write_test_data_file(self, basename: str,
-                             test_cases: Iterable[test_case.TestCase]) -> None:
-        """Write the test cases to a .data file.
-
-        The output file is ``basename + '.data'`` in the test suite directory.
-        """
-        filename = self.filename_for(basename)
-        test_case.write_data_file(filename, test_cases)
-
     # Note that targets whose names contain 'test_format' have their content
     # validated by `abi_check.py`.
-    TARGETS = {
+    targets = {
         'test_suite_psa_crypto_generate_key.generated':
         lambda info: KeyGenerate(info).test_cases_for_key_generation(),
         'test_suite_psa_crypto_not_supported.generated':
@@ -938,44 +909,12 @@ class TestGenerator:
         lambda info: StorageFormatV0(info).all_test_cases(),
     } #type: Dict[str, Callable[[Information], Iterable[test_case.TestCase]]]
 
-    def generate_target(self, name: str) -> None:
+    def __init__(self, options):
-        test_cases = self.TARGETS[name](self.info)
+        super().__init__(options)
-        self.write_test_data_file(name, test_cases)
+        self.info = Information()
 
-def main(args):
+    def generate_target(self, name: str, *target_args) -> None:
-    """Command line entry point."""
+        super().generate_target(name, self.info)
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--list', action='store_true',
-                        help='List available targets and exit')
-    parser.add_argument('--list-for-cmake', action='store_true',
-                        help='Print \';\'-separated list of available targets and exit')
-    parser.add_argument('--directory', metavar='DIR',
-                        help='Output directory (default: tests/suites)')
-    parser.add_argument('targets', nargs='*', metavar='TARGET',
-                        help='Target file to generate (default: all; "-": none)')
-    options = parser.parse_args(args)
-    build_tree.chdir_to_root()
-    generator = TestGenerator(options)
-    if options.list:
-        for name in sorted(generator.TARGETS):
-            print(generator.filename_for(name))
-        return
-    # List in a cmake list format (i.e. ';'-separated)
-    if options.list_for_cmake:
-        print(';'.join(generator.filename_for(name)
-                       for name in sorted(generator.TARGETS)), end='')
-        return
-    if options.targets:
-        # Allow "-" as a special case so you can run
-        # ``generate_psa_tests.py - $targets`` and it works uniformly whether
-        # ``$targets`` is empty or not.
-        options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
-                           for target in options.targets
-                           if target != '-']
-    else:
-        options.targets = sorted(generator.TARGETS)
-    for target in options.targets:
-        generator.generate_target(target)
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    test_generation.main(sys.argv[1:], __doc__, PSATestGenerator)