Merge pull request #6093 from wernerlewis/bignum_test_script
Add bignum test case generation script
This commit is contained in:
Gilles Peskine
GitHub
commit
1716f06ee3
7 changed files with 555 additions and 92 deletions
|
|
@ -10,4 +10,5 @@ perl scripts\generate_features.pl || exit /b 1
|
|||
python scripts\generate_ssl_debug_helpers.py || exit /b 1
|
||||
perl scripts\generate_visualc_files.pl || exit /b 1
|
||||
python scripts\generate_psa_constants.py || exit /b 1
|
||||
python tests\scripts\generate_bignum_tests.py || exit /b 1
|
||||
python tests\scripts\generate_psa_tests.py || exit /b 1
|
||||
|
|
|
|||
219
scripts/mbedtls_dev/test_generation.py
Normal file
219
scripts/mbedtls_dev/test_generation.py
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
"""Common test generation classes and main function.
|
||||
|
||||
These are used both by generate_psa_tests.py and generate_bignum_tests.py.
|
||||
"""
|
||||
|
||||
# Copyright The Mbed TLS Contributors
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar
|
||||
|
||||
from mbedtls_dev import build_tree
|
||||
from mbedtls_dev import test_case
|
||||
|
||||
T = TypeVar('T') #pylint: disable=invalid-name
|
||||
|
||||
|
||||
class BaseTarget(metaclass=ABCMeta):
|
||||
"""Base target for test case generation.
|
||||
|
||||
Child classes of this class represent an output file, and can be referred
|
||||
to as file targets. These indicate where test cases will be written to for
|
||||
all subclasses of the file target, which is set by `target_basename`.
|
||||
|
||||
Attributes:
|
||||
count: Counter for test cases from this class.
|
||||
case_description: Short description of the test case. This may be
|
||||
automatically generated using the class, or manually set.
|
||||
dependencies: A list of dependencies required for the test case.
|
||||
show_test_count: Toggle for inclusion of `count` in the test description.
|
||||
target_basename: Basename of file to write generated tests to. This
|
||||
should be specified in a child class of BaseTarget.
|
||||
test_function: Test function which the class generates cases for.
|
||||
test_name: A common name or description of the test function. This can
|
||||
be `test_function`, a clearer equivalent, or a short summary of the
|
||||
test function's purpose.
|
||||
"""
|
||||
count = 0
|
||||
case_description = ""
|
||||
dependencies = [] # type: List[str]
|
||||
show_test_count = True
|
||||
target_basename = ""
|
||||
test_function = ""
|
||||
test_name = ""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
# pylint: disable=unused-argument
|
||||
cls.count += 1
|
||||
return super().__new__(cls)
|
||||
|
||||
@abstractmethod
|
||||
def arguments(self) -> List[str]:
|
||||
"""Get the list of arguments for the test case.
|
||||
|
||||
Override this method to provide the list of arguments required for
|
||||
the `test_function`.
|
||||
|
||||
Returns:
|
||||
List of arguments required for the test function.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def description(self) -> str:
|
||||
"""Create a test case description.
|
||||
|
||||
Creates a description of the test case, including a name for the test
|
||||
function, an optional case count, and a description of the specific
|
||||
test case. This should inform a reader what is being tested, and
|
||||
provide context for the test case.
|
||||
|
||||
Returns:
|
||||
Description for the test case.
|
||||
"""
|
||||
if self.show_test_count:
|
||||
return "{} #{} {}".format(
|
||||
self.test_name, self.count, self.case_description
|
||||
).strip()
|
||||
else:
|
||||
return "{} {}".format(self.test_name, self.case_description).strip()
|
||||
|
||||
|
||||
def create_test_case(self) -> test_case.TestCase:
|
||||
"""Generate TestCase from the instance."""
|
||||
tc = test_case.TestCase()
|
||||
tc.set_description(self.description())
|
||||
tc.set_function(self.test_function)
|
||||
tc.set_arguments(self.arguments())
|
||||
tc.set_dependencies(self.dependencies)
|
||||
|
||||
return tc
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
|
||||
"""Generate test cases for the class test function.
|
||||
|
||||
This will be called in classes where `test_function` is set.
|
||||
Implementations should yield TestCase objects, by creating instances
|
||||
of the class with appropriate input data, and then calling
|
||||
`create_test_case()` on each.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def generate_tests(cls) -> Iterator[test_case.TestCase]:
|
||||
"""Generate test cases for the class and its subclasses.
|
||||
|
||||
In classes with `test_function` set, `generate_function_tests()` is
|
||||
called to generate test cases first.
|
||||
|
||||
In all classes, this method will iterate over its subclasses, and
|
||||
yield from `generate_tests()` in each. Calling this method on a class X
|
||||
will yield test cases from all classes derived from X.
|
||||
"""
|
||||
if cls.test_function:
|
||||
yield from cls.generate_function_tests()
|
||||
for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__):
|
||||
yield from subclass.generate_tests()
|
||||
|
||||
|
||||
class TestGenerator:
|
||||
"""Generate test cases and write to data files."""
|
||||
def __init__(self, options) -> None:
|
||||
self.test_suite_directory = self.get_option(options, 'directory',
|
||||
'tests/suites')
|
||||
# Update `targets` with an entry for each child class of BaseTarget.
|
||||
# Each entry represents a file generated by the BaseTarget framework,
|
||||
# and enables generating the .data files using the CLI.
|
||||
self.targets.update({
|
||||
subclass.target_basename: subclass.generate_tests
|
||||
for subclass in BaseTarget.__subclasses__()
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
def get_option(options, name: str, default: T) -> T:
|
||||
value = getattr(options, name, None)
|
||||
return default if value is None else value
|
||||
|
||||
def filename_for(self, basename: str) -> str:
|
||||
"""The location of the data file with the specified base name."""
|
||||
return posixpath.join(self.test_suite_directory, basename + '.data')
|
||||
|
||||
def write_test_data_file(self, basename: str,
|
||||
test_cases: Iterable[test_case.TestCase]) -> None:
|
||||
"""Write the test cases to a .data file.
|
||||
|
||||
The output file is ``basename + '.data'`` in the test suite directory.
|
||||
"""
|
||||
filename = self.filename_for(basename)
|
||||
test_case.write_data_file(filename, test_cases)
|
||||
|
||||
# Note that targets whose names contain 'test_format' have their content
|
||||
# validated by `abi_check.py`.
|
||||
targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
|
||||
|
||||
def generate_target(self, name: str, *target_args) -> None:
|
||||
"""Generate cases and write to data file for a target.
|
||||
|
||||
For target callables which require arguments, override this function
|
||||
and pass these arguments using super() (see PSATestGenerator).
|
||||
"""
|
||||
test_cases = self.targets[name](*target_args)
|
||||
self.write_test_data_file(name, test_cases)
|
||||
|
||||
def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
|
||||
"""Command line entry point."""
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('--list', action='store_true',
|
||||
help='List available targets and exit')
|
||||
parser.add_argument('--list-for-cmake', action='store_true',
|
||||
help='Print \';\'-separated list of available targets and exit')
|
||||
parser.add_argument('--directory', metavar='DIR',
|
||||
help='Output directory (default: tests/suites)')
|
||||
# The `--directory` option is interpreted relative to the directory from
|
||||
# which the script is invoked, but the default is relative to the root of
|
||||
# the mbedtls tree. The default should not be set above, but instead after
|
||||
# `build_tree.chdir_to_root()` is called.
|
||||
parser.add_argument('targets', nargs='*', metavar='TARGET',
|
||||
help='Target file to generate (default: all; "-": none)')
|
||||
options = parser.parse_args(args)
|
||||
build_tree.chdir_to_root()
|
||||
generator = generator_class(options)
|
||||
if options.list:
|
||||
for name in sorted(generator.targets):
|
||||
print(generator.filename_for(name))
|
||||
return
|
||||
# List in a cmake list format (i.e. ';'-separated)
|
||||
if options.list_for_cmake:
|
||||
print(';'.join(generator.filename_for(name)
|
||||
for name in sorted(generator.targets)), end='')
|
||||
return
|
||||
if options.targets:
|
||||
# Allow "-" as a special case so you can run
|
||||
# ``generate_xxx_tests.py - $targets`` and it works uniformly whether
|
||||
# ``$targets`` is empty or not.
|
||||
options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
|
||||
for target in options.targets
|
||||
if target != '-']
|
||||
else:
|
||||
options.targets = sorted(generator.targets)
|
||||
for target in options.targets:
|
||||
generator.generate_target(target)
|
||||
|
|
@ -17,6 +17,17 @@ endif()
|
|||
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/suites)
|
||||
|
||||
# Get base names for generated files (starting at "suites/")
|
||||
execute_process(
|
||||
COMMAND
|
||||
${MBEDTLS_PYTHON_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
|
||||
--list-for-cmake
|
||||
--directory suites
|
||||
WORKING_DIRECTORY
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/..
|
||||
OUTPUT_VARIABLE
|
||||
base_bignum_generated_data_files)
|
||||
|
||||
execute_process(
|
||||
COMMAND
|
||||
${MBEDTLS_PYTHON_EXECUTABLE}
|
||||
|
|
@ -26,18 +37,37 @@ execute_process(
|
|||
WORKING_DIRECTORY
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/..
|
||||
OUTPUT_VARIABLE
|
||||
base_generated_data_files)
|
||||
base_psa_generated_data_files)
|
||||
|
||||
# Derive generated file paths in the build directory
|
||||
set(generated_data_files "")
|
||||
foreach(file ${base_generated_data_files})
|
||||
list(APPEND generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
|
||||
set(base_generated_data_files ${base_bignum_generated_data_files} ${base_psa_generated_data_files})
|
||||
set(bignum_generated_data_files "")
|
||||
set(psa_generated_data_files "")
|
||||
foreach(file ${base_bignum_generated_data_files})
|
||||
list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
|
||||
endforeach()
|
||||
foreach(file ${base_psa_generated_data_files})
|
||||
list(APPEND psa_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
|
||||
endforeach()
|
||||
|
||||
if(GEN_FILES)
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${generated_data_files}
|
||||
${bignum_generated_data_files}
|
||||
WORKING_DIRECTORY
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/..
|
||||
COMMAND
|
||||
${MBEDTLS_PYTHON_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
|
||||
--directory ${CMAKE_CURRENT_BINARY_DIR}/suites
|
||||
DEPENDS
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py
|
||||
)
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${psa_generated_data_files}
|
||||
WORKING_DIRECTORY
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/..
|
||||
COMMAND
|
||||
|
|
@ -50,6 +80,7 @@ if(GEN_FILES)
|
|||
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/macro_collector.py
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/psa_storage.py
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_config.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_values.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_extra.h
|
||||
|
|
@ -65,7 +96,8 @@ endif()
|
|||
# they can cause race conditions in parallel builds.
|
||||
# With this line, only 4 sub-makefiles include the above command, that reduces
|
||||
# the risk of a race.
|
||||
add_custom_target(test_suite_generated_data DEPENDS ${generated_data_files})
|
||||
add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files})
|
||||
add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files})
|
||||
# Test suites caught by SKIP_TEST_SUITES are built but not executed.
|
||||
# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
|
||||
# but not "test_suite_foobar".
|
||||
|
|
@ -82,23 +114,39 @@ function(add_test_suite suite_name)
|
|||
|
||||
# Get the test names of the tests with generated .data files
|
||||
# from the generated_data_files list in parent scope.
|
||||
set(generated_data_names "")
|
||||
foreach(generated_data_file ${generated_data_files})
|
||||
set(bignum_generated_data_names "")
|
||||
set(psa_generated_data_names "")
|
||||
foreach(generated_data_file ${bignum_generated_data_files})
|
||||
# Get the plain filename
|
||||
get_filename_component(generated_data_name ${generated_data_file} NAME)
|
||||
# Remove the ".data" extension
|
||||
get_name_without_last_ext(generated_data_name ${generated_data_name})
|
||||
# Remove leading "test_suite_"
|
||||
string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
|
||||
list(APPEND generated_data_names ${generated_data_name})
|
||||
list(APPEND bignum_generated_data_names ${generated_data_name})
|
||||
endforeach()
|
||||
foreach(generated_data_file ${psa_generated_data_files})
|
||||
# Get the plain filename
|
||||
get_filename_component(generated_data_name ${generated_data_file} NAME)
|
||||
# Remove the ".data" extension
|
||||
get_name_without_last_ext(generated_data_name ${generated_data_name})
|
||||
# Remove leading "test_suite_"
|
||||
string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
|
||||
list(APPEND psa_generated_data_names ${generated_data_name})
|
||||
endforeach()
|
||||
|
||||
if(";${generated_data_names};" MATCHES ";${data_name};")
|
||||
if(";${bignum_generated_data_names};" MATCHES ";${data_name};")
|
||||
set(data_file
|
||||
${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
|
||||
set(dependency test_suite_bignum_generated_data)
|
||||
elseif(";${psa_generated_data_names};" MATCHES ";${data_name};")
|
||||
set(data_file
|
||||
${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
|
||||
set(dependency test_suite_psa_generated_data)
|
||||
else()
|
||||
set(data_file
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data)
|
||||
set(dependency test_suite_bignum_generated_data test_suite_psa_generated_data)
|
||||
endif()
|
||||
|
||||
add_custom_command(
|
||||
|
|
@ -129,7 +177,7 @@ function(add_test_suite suite_name)
|
|||
)
|
||||
|
||||
add_executable(test_suite_${data_name} test_suite_${data_name}.c $<TARGET_OBJECTS:mbedtls_test>)
|
||||
add_dependencies(test_suite_${data_name} test_suite_generated_data)
|
||||
add_dependencies(test_suite_${data_name} ${dependency})
|
||||
target_link_libraries(test_suite_${data_name} ${libs})
|
||||
# Include test-specific header files from ./include and private header
|
||||
# files (used by some invasive tests) from ../library. Public header
|
||||
|
|
|
|||
|
|
@ -66,28 +66,45 @@ PYTHON ?= $(shell if type python3 >/dev/null 2>/dev/null; then echo python3; els
|
|||
endif
|
||||
|
||||
.PHONY: generated_files
|
||||
GENERATED_DATA_FILES := $(patsubst tests/%,%,$(shell \
|
||||
GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \
|
||||
$(PYTHON) scripts/generate_bignum_tests.py --list || \
|
||||
echo FAILED \
|
||||
))
|
||||
ifeq ($(GENERATED_BIGNUM_DATA_FILES),FAILED)
|
||||
$(error "$(PYTHON) scripts/generate_bignum_tests.py --list" failed)
|
||||
endif
|
||||
GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \
|
||||
$(PYTHON) scripts/generate_psa_tests.py --list || \
|
||||
echo FAILED \
|
||||
))
|
||||
ifeq ($(GENERATED_DATA_FILES),FAILED)
|
||||
ifeq ($(GENERATED_PSA_DATA_FILES),FAILED)
|
||||
$(error "$(PYTHON) scripts/generate_psa_tests.py --list" failed)
|
||||
endif
|
||||
GENERATED_FILES := $(GENERATED_DATA_FILES)
|
||||
GENERATED_FILES := $(GENERATED_PSA_DATA_FILES) $(GENERATED_BIGNUM_DATA_FILES)
|
||||
generated_files: $(GENERATED_FILES)
|
||||
|
||||
# generate_psa_tests.py spends more time analyzing inputs than generating
|
||||
# outputs. Its inputs are the same no matter which files are being generated.
|
||||
# generate_bignum_tests.py and generate_psa_tests.py spend more time analyzing
|
||||
# inputs than generating outputs. Its inputs are the same no matter which files
|
||||
# are being generated.
|
||||
# It's rare not to want all the outputs. So always generate all of its outputs.
|
||||
# Use an intermediate phony dependency so that parallel builds don't run
|
||||
# a separate instance of the recipe for each output file.
|
||||
.SECONDARY: generated_psa_test_data
|
||||
$(GENERATED_DATA_FILES): generated_psa_test_data
|
||||
.SECONDARY: generated_bignum_test_data generated_psa_test_data
|
||||
$(GENERATED_BIGNUM_DATA_FILES): generated_bignum_test_data
|
||||
generated_bignum_test_data: scripts/generate_bignum_tests.py
|
||||
generated_bignum_test_data: ../scripts/mbedtls_dev/test_case.py
|
||||
generated_bignum_test_data: ../scripts/mbedtls_dev/test_generation.py
|
||||
generated_bignum_test_data:
|
||||
echo " Gen $(GENERATED_BIGNUM_DATA_FILES)"
|
||||
$(PYTHON) scripts/generate_bignum_tests.py
|
||||
|
||||
$(GENERATED_PSA_DATA_FILES): generated_psa_test_data
|
||||
generated_psa_test_data: scripts/generate_psa_tests.py
|
||||
generated_psa_test_data: ../scripts/mbedtls_dev/crypto_knowledge.py
|
||||
generated_psa_test_data: ../scripts/mbedtls_dev/macro_collector.py
|
||||
generated_psa_test_data: ../scripts/mbedtls_dev/psa_storage.py
|
||||
generated_psa_test_data: ../scripts/mbedtls_dev/test_case.py
|
||||
generated_psa_test_data: ../scripts/mbedtls_dev/test_generation.py
|
||||
## The generated file only depends on the options that are present in
|
||||
## crypto_config.h, not on which options are set. To avoid regenerating this
|
||||
## file all the time when switching between configurations, don't declare
|
||||
|
|
@ -98,7 +115,7 @@ generated_psa_test_data: ../include/psa/crypto_values.h
|
|||
generated_psa_test_data: ../include/psa/crypto_extra.h
|
||||
generated_psa_test_data: suites/test_suite_psa_crypto_metadata.data
|
||||
generated_psa_test_data:
|
||||
echo " Gen $(GENERATED_DATA_FILES) ..."
|
||||
echo " Gen $(GENERATED_PSA_DATA_FILES) ..."
|
||||
$(PYTHON) scripts/generate_psa_tests.py
|
||||
|
||||
# A test application is built for each suites/test_suite_*.data file.
|
||||
|
|
@ -107,7 +124,7 @@ generated_psa_test_data:
|
|||
DATA_FILES := $(wildcard suites/test_suite_*.data)
|
||||
# Make sure that generated data files are included even if they don't
|
||||
# exist yet when the makefile is parsed.
|
||||
DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_DATA_FILES))
|
||||
DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_FILES))
|
||||
APPS = $(basename $(subst suites/,,$(DATA_FILES)))
|
||||
|
||||
# Construct executable name by adding OS specific suffix $(EXEXT).
|
||||
|
|
|
|||
|
|
@ -126,4 +126,5 @@ check scripts/generate_ssl_debug_helpers.py library/ssl_debug_helpers_generated.
|
|||
# the step that creates or updates these files.
|
||||
check scripts/generate_visualc_files.pl visualc/VS2010
|
||||
check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c
|
||||
check tests/scripts/generate_bignum_tests.py $(tests/scripts/generate_bignum_tests.py --list)
|
||||
check tests/scripts/generate_psa_tests.py $(tests/scripts/generate_psa_tests.py --list)
|
||||
|
|
|
|||
238
tests/scripts/generate_bignum_tests.py
Executable file
238
tests/scripts/generate_bignum_tests.py
Executable file
|
|
@ -0,0 +1,238 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Generate test data for bignum functions.
|
||||
|
||||
With no arguments, generate all test data. With non-option arguments,
|
||||
generate only the specified files.
|
||||
|
||||
Class structure:
|
||||
|
||||
Child classes of test_generation.BaseTarget (file targets) represent an output
|
||||
file. These indicate where test cases will be written to, for all subclasses of
|
||||
this target. Multiple file targets should not reuse a `target_basename`.
|
||||
|
||||
Each subclass derived from a file target can either be:
|
||||
- A concrete class, representing a test function, which generates test cases.
|
||||
- An abstract class containing shared methods and attributes, not associated
|
||||
with a test function. An example is BignumOperation, which provides
|
||||
common features used for bignum binary operations.
|
||||
|
||||
Both concrete and abstract subclasses can be derived from, to implement
|
||||
additional test cases (see BignumCmp and BignumCmpAbs for examples of deriving
|
||||
from abstract and concrete classes).
|
||||
|
||||
|
||||
Adding test case generation for a function:
|
||||
|
||||
A subclass representing the test function should be added, deriving from a
|
||||
file target such as BignumTarget. This test class must set/implement the
|
||||
following:
|
||||
- test_function: the function name from the associated .function file.
|
||||
- test_name: a descriptive name or brief summary to refer to the test
|
||||
function.
|
||||
- arguments(): a method to generate the list of arguments required for the
|
||||
test_function.
|
||||
- generate_function_test(): a method to generate TestCases for the function.
|
||||
This should create instances of the class with required input data, and
|
||||
call `.create_test_case()` to yield the TestCase.
|
||||
|
||||
Additional details and other attributes/methods are given in the documentation
|
||||
of BaseTarget in test_generation.py.
|
||||
"""
|
||||
|
||||
# Copyright The Mbed TLS Contributors
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import itertools
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from typing import Iterator, List, Tuple, TypeVar
|
||||
|
||||
import scripts_path # pylint: disable=unused-import
|
||||
from mbedtls_dev import test_case
|
||||
from mbedtls_dev import test_generation
|
||||
|
||||
T = TypeVar('T') #pylint: disable=invalid-name
|
||||
|
||||
def hex_to_int(val: str) -> int:
|
||||
return int(val, 16) if val else 0
|
||||
|
||||
def quote_str(val) -> str:
|
||||
return "\"{}\"".format(val)
|
||||
|
||||
def combination_pairs(values: List[T]) -> List[Tuple[T, T]]:
|
||||
"""Return all pair combinations from input values.
|
||||
|
||||
The return value is cast, as older versions of mypy are unable to derive
|
||||
the specific type returned by itertools.combinations_with_replacement.
|
||||
"""
|
||||
return typing.cast(
|
||||
List[Tuple[T, T]],
|
||||
list(itertools.combinations_with_replacement(values, 2))
|
||||
)
|
||||
|
||||
|
||||
class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta):
|
||||
#pylint: disable=abstract-method
|
||||
"""Target for bignum (mpi) test case generation."""
|
||||
target_basename = 'test_suite_mpi.generated'
|
||||
|
||||
|
||||
class BignumOperation(BignumTarget, metaclass=ABCMeta):
|
||||
"""Common features for bignum binary operations.
|
||||
|
||||
This adds functionality common in binary operation tests. This includes
|
||||
generation of case descriptions, using descriptions of values and symbols
|
||||
to represent the operation or result.
|
||||
|
||||
Attributes:
|
||||
symbol: Symbol used for the operation in case description.
|
||||
input_values: List of values to use as test case inputs. These are
|
||||
combined to produce pairs of values.
|
||||
input_cases: List of tuples containing pairs of test case inputs. This
|
||||
can be used to implement specific pairs of inputs.
|
||||
"""
|
||||
symbol = ""
|
||||
input_values = [
|
||||
"", "0", "7b", "-7b",
|
||||
"0000000000000000123", "-0000000000000000123",
|
||||
"1230000000000000000", "-1230000000000000000"
|
||||
] # type: List[str]
|
||||
input_cases = [] # type: List[Tuple[str, str]]
|
||||
|
||||
def __init__(self, val_a: str, val_b: str) -> None:
|
||||
self.arg_a = val_a
|
||||
self.arg_b = val_b
|
||||
self.int_a = hex_to_int(val_a)
|
||||
self.int_b = hex_to_int(val_b)
|
||||
|
||||
def arguments(self) -> List[str]:
|
||||
return [quote_str(self.arg_a), quote_str(self.arg_b), self.result()]
|
||||
|
||||
def description(self) -> str:
|
||||
"""Generate a description for the test case.
|
||||
|
||||
If not set, case_description uses the form A `symbol` B, where symbol
|
||||
is used to represent the operation. Descriptions of each value are
|
||||
generated to provide some context to the test case.
|
||||
"""
|
||||
if not self.case_description:
|
||||
self.case_description = "{} {} {}".format(
|
||||
self.value_description(self.arg_a),
|
||||
self.symbol,
|
||||
self.value_description(self.arg_b)
|
||||
)
|
||||
return super().description()
|
||||
|
||||
@abstractmethod
|
||||
def result(self) -> str:
|
||||
"""Get the result of the operation.
|
||||
|
||||
This could be calculated during initialization and stored as `_result`
|
||||
and then returned, or calculated when the method is called.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def value_description(val) -> str:
|
||||
"""Generate a description of the argument val.
|
||||
|
||||
This produces a simple description of the value, which is used in test
|
||||
case naming to add context.
|
||||
"""
|
||||
if val == "":
|
||||
return "0 (null)"
|
||||
if val == "0":
|
||||
return "0 (1 limb)"
|
||||
|
||||
if val[0] == "-":
|
||||
tmp = "negative"
|
||||
val = val[1:]
|
||||
else:
|
||||
tmp = "positive"
|
||||
if val[0] == "0":
|
||||
tmp += " with leading zero limb"
|
||||
elif len(val) > 10:
|
||||
tmp = "large " + tmp
|
||||
return tmp
|
||||
|
||||
@classmethod
|
||||
def get_value_pairs(cls) -> Iterator[Tuple[str, str]]:
|
||||
"""Generator to yield pairs of inputs.
|
||||
|
||||
Combinations are first generated from all input values, and then
|
||||
specific cases provided.
|
||||
"""
|
||||
yield from combination_pairs(cls.input_values)
|
||||
yield from cls.input_cases
|
||||
|
||||
@classmethod
|
||||
def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
|
||||
for a_value, b_value in cls.get_value_pairs():
|
||||
cur_op = cls(a_value, b_value)
|
||||
yield cur_op.create_test_case()
|
||||
|
||||
|
||||
class BignumCmp(BignumOperation):
|
||||
"""Test cases for bignum value comparison."""
|
||||
count = 0
|
||||
test_function = "mbedtls_mpi_cmp_mpi"
|
||||
test_name = "MPI compare"
|
||||
input_cases = [
|
||||
("-2", "-3"),
|
||||
("-2", "-2"),
|
||||
("2b4", "2b5"),
|
||||
("2b5", "2b6")
|
||||
]
|
||||
|
||||
def __init__(self, val_a, val_b) -> None:
|
||||
super().__init__(val_a, val_b)
|
||||
self._result = int(self.int_a > self.int_b) - int(self.int_a < self.int_b)
|
||||
self.symbol = ["<", "==", ">"][self._result + 1]
|
||||
|
||||
def result(self) -> str:
|
||||
return str(self._result)
|
||||
|
||||
|
||||
class BignumCmpAbs(BignumCmp):
|
||||
"""Test cases for absolute bignum value comparison."""
|
||||
count = 0
|
||||
test_function = "mbedtls_mpi_cmp_abs"
|
||||
test_name = "MPI compare (abs)"
|
||||
|
||||
def __init__(self, val_a, val_b) -> None:
|
||||
super().__init__(val_a.strip("-"), val_b.strip("-"))
|
||||
|
||||
|
||||
class BignumAdd(BignumOperation):
|
||||
"""Test cases for bignum value addition."""
|
||||
count = 0
|
||||
symbol = "+"
|
||||
test_function = "mbedtls_mpi_add_mpi"
|
||||
test_name = "MPI add"
|
||||
input_cases = combination_pairs(
|
||||
[
|
||||
"1c67967269c6", "9cde3",
|
||||
"-1c67967269c6", "-9cde3",
|
||||
]
|
||||
)
|
||||
|
||||
def result(self) -> str:
|
||||
return quote_str("{:x}".format(self.int_a + self.int_b))
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Use the section of the docstring relevant to the CLI as description
|
||||
test_generation.main(sys.argv[1:], "\n".join(__doc__.splitlines()[:4]))
|
||||
|
|
@ -20,22 +20,17 @@ generate only the specified files.
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import enum
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sys
|
||||
from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional, TypeVar
|
||||
from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional
|
||||
|
||||
import scripts_path # pylint: disable=unused-import
|
||||
from mbedtls_dev import build_tree
|
||||
from mbedtls_dev import crypto_knowledge
|
||||
from mbedtls_dev import macro_collector
|
||||
from mbedtls_dev import psa_storage
|
||||
from mbedtls_dev import test_case
|
||||
|
||||
T = TypeVar('T') #pylint: disable=invalid-name
|
||||
from mbedtls_dev import test_generation
|
||||
|
||||
|
||||
def psa_want_symbol(name: str) -> str:
|
||||
|
|
@ -897,35 +892,11 @@ class StorageFormatV0(StorageFormat):
|
|||
yield from super().generate_all_keys()
|
||||
yield from self.all_keys_for_implicit_usage()
|
||||
|
||||
class TestGenerator:
|
||||
"""Generate test data."""
|
||||
|
||||
def __init__(self, options) -> None:
|
||||
self.test_suite_directory = self.get_option(options, 'directory',
|
||||
'tests/suites')
|
||||
self.info = Information()
|
||||
|
||||
@staticmethod
|
||||
def get_option(options, name: str, default: T) -> T:
|
||||
value = getattr(options, name, None)
|
||||
return default if value is None else value
|
||||
|
||||
def filename_for(self, basename: str) -> str:
|
||||
"""The location of the data file with the specified base name."""
|
||||
return posixpath.join(self.test_suite_directory, basename + '.data')
|
||||
|
||||
def write_test_data_file(self, basename: str,
|
||||
test_cases: Iterable[test_case.TestCase]) -> None:
|
||||
"""Write the test cases to a .data file.
|
||||
|
||||
The output file is ``basename + '.data'`` in the test suite directory.
|
||||
"""
|
||||
filename = self.filename_for(basename)
|
||||
test_case.write_data_file(filename, test_cases)
|
||||
|
||||
class PSATestGenerator(test_generation.TestGenerator):
|
||||
"""Test generator subclass including PSA targets and info."""
|
||||
# Note that targets whose names contain 'test_format' have their content
|
||||
# validated by `abi_check.py`.
|
||||
TARGETS = {
|
||||
targets = {
|
||||
'test_suite_psa_crypto_generate_key.generated':
|
||||
lambda info: KeyGenerate(info).test_cases_for_key_generation(),
|
||||
'test_suite_psa_crypto_not_supported.generated':
|
||||
|
|
@ -938,44 +909,12 @@ class TestGenerator:
|
|||
lambda info: StorageFormatV0(info).all_test_cases(),
|
||||
} #type: Dict[str, Callable[[Information], Iterable[test_case.TestCase]]]
|
||||
|
||||
def generate_target(self, name: str) -> None:
|
||||
test_cases = self.TARGETS[name](self.info)
|
||||
self.write_test_data_file(name, test_cases)
|
||||
def __init__(self, options):
|
||||
super().__init__(options)
|
||||
self.info = Information()
|
||||
|
||||
def main(args):
|
||||
"""Command line entry point."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--list', action='store_true',
|
||||
help='List available targets and exit')
|
||||
parser.add_argument('--list-for-cmake', action='store_true',
|
||||
help='Print \';\'-separated list of available targets and exit')
|
||||
parser.add_argument('--directory', metavar='DIR',
|
||||
help='Output directory (default: tests/suites)')
|
||||
parser.add_argument('targets', nargs='*', metavar='TARGET',
|
||||
help='Target file to generate (default: all; "-": none)')
|
||||
options = parser.parse_args(args)
|
||||
build_tree.chdir_to_root()
|
||||
generator = TestGenerator(options)
|
||||
if options.list:
|
||||
for name in sorted(generator.TARGETS):
|
||||
print(generator.filename_for(name))
|
||||
return
|
||||
# List in a cmake list format (i.e. ';'-separated)
|
||||
if options.list_for_cmake:
|
||||
print(';'.join(generator.filename_for(name)
|
||||
for name in sorted(generator.TARGETS)), end='')
|
||||
return
|
||||
if options.targets:
|
||||
# Allow "-" as a special case so you can run
|
||||
# ``generate_psa_tests.py - $targets`` and it works uniformly whether
|
||||
# ``$targets`` is empty or not.
|
||||
options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
|
||||
for target in options.targets
|
||||
if target != '-']
|
||||
else:
|
||||
options.targets = sorted(generator.TARGETS)
|
||||
for target in options.targets:
|
||||
generator.generate_target(target)
|
||||
def generate_target(self, name: str, *target_args) -> None:
|
||||
super().generate_target(name, self.info)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
test_generation.main(sys.argv[1:], __doc__, PSATestGenerator)
|
||||
|
|
|
|||
Loading…
Reference in a new issue