Test program for psa_constant_names
Test psa_constant_names on many inputs. For each input, find out the numerical value by compiling and running a C program, pass the numerical value to psa_constant_names and compare the output with the original input. Gather inputs by parsing psa/crypto.h and test_suite_psa_crypto_metadata.data. For macros that take an argument, list some possible arguments using the parsed data.
This commit is contained in:
Gilles Peskine
Darryl Green
parent
567840e335
commit
2482702d15
1 changed files with 241 additions and 0 deletions
241
tests/scripts/test_psa_constant_names.py
Executable file
241
tests/scripts/test_psa_constant_names.py
Executable file
|
|
@ -0,0 +1,241 @@
|
|||
#!/usr/bin/env python3
|
||||
'''Test the program psa_constant_names.
|
||||
Gather constant names from header files and test cases. Compile a C program
|
||||
to print out their numerical values, feed these numerical values to
|
||||
psa_constant_names, and check that the output is the original name.
|
||||
Return 0 if all test cases pass, 1 if the output was not always as expected,
|
||||
or 1 (with a Python backtrace) if there was an operational error.'''
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
class Inputs:
|
||||
'''Accumulate information about macros to test.
|
||||
This includes macro names as well as information about their arguments
|
||||
when applicable.'''
|
||||
def __init__(self):
|
||||
# Sets of names per type
|
||||
self.statuses = set(['PSA_SUCCESS'])
|
||||
self.algorithms = set(['0xffffffff'])
|
||||
self.ecc_curves = set(['0xffff'])
|
||||
self.key_types = set(['0xffffffff'])
|
||||
self.key_usage_flags = set(['0x80000000'])
|
||||
# Hard-coded value for an unknown hash algorithm
|
||||
self.hash_algorithms = set(['0x010000ff'])
|
||||
# Identifier prefixes
|
||||
self.table_by_prefix = {
|
||||
'ERROR': self.statuses,
|
||||
'ALG': self.algorithms,
|
||||
'CURVE': self.ecc_curves,
|
||||
'KEY_TYPE': self.key_types,
|
||||
'KEY_USAGE': self.key_usage_flags,
|
||||
}
|
||||
# macro name -> list of argument names
|
||||
self.argspecs = {}
|
||||
# argument name -> list of values
|
||||
self.arguments_for = {}
|
||||
|
||||
def gather_arguments(self):
|
||||
'''Populate the list of values for macro arguments.
|
||||
Call this after parsing all the inputs.'''
|
||||
self.arguments_for['hash_alg'] = sorted(self.hash_algorithms)
|
||||
self.arguments_for['curve'] = sorted(self.ecc_curves)
|
||||
|
||||
def format_arguments(self, name, arguments):
|
||||
'''Format a macro call with arguments..'''
|
||||
return name + '(' + ', '.join(arguments) + ')'
|
||||
|
||||
def distribute_arguments(self, name):
|
||||
'''Generate macro calls with each tested argument set.
|
||||
If name is a macro without arguments, just yield "name".
|
||||
If name is a macro with arguments, yield a series of "name(arg1,...,argN)"
|
||||
where each argument takes each possible value at least once.'''
|
||||
if name not in self.argspecs:
|
||||
yield name
|
||||
return
|
||||
argspec = self.argspecs[name]
|
||||
if argspec == []:
|
||||
yield name + '()'
|
||||
return
|
||||
argument_lists = [self.arguments_for[arg] for arg in argspec]
|
||||
arguments = [values[0] for values in argument_lists]
|
||||
yield self.format_arguments(name, arguments)
|
||||
for i in range(len(arguments)):
|
||||
for value in argument_lists[i][1:]:
|
||||
arguments[i] = value
|
||||
yield self.format_arguments(name, arguments)
|
||||
arguments[i] = argument_lists[0]
|
||||
|
||||
# Regex for interesting header lines.
|
||||
# Groups: 1=macro name, 2=type, 3=argument list (optional).
|
||||
header_line_re = \
|
||||
re.compile(r'#define +' +
|
||||
r'(PSA_((?:KEY_)?[A-Z]+)_\w+)' +
|
||||
r'(?:\(([^\n()]*)\))?')
|
||||
# Regex of macro names to exclude.
|
||||
excluded_name_re = re.compile('_(?:GET|IS|OF)_|_(?:BASE|FLAG|MASK)\Z')
|
||||
argument_split_re = re.compile(r' *, *')
|
||||
def parse_header_line(self, line):
|
||||
'''Parse a C header line, looking for "#define PSA_xxx".'''
|
||||
m = re.match(self.header_line_re, line)
|
||||
if not m:
|
||||
return
|
||||
name = m.group(1)
|
||||
if re.search(self.excluded_name_re, name):
|
||||
return
|
||||
dest = self.table_by_prefix.get(m.group(2))
|
||||
if dest is None:
|
||||
return
|
||||
dest.add(name)
|
||||
if m.group(3):
|
||||
self.argspecs[name] = re.split(self.argument_split_re, m.group(3))
|
||||
|
||||
def parse_header(self, filename):
|
||||
'''Parse a C header file, looking for "#define PSA_xxx".'''
|
||||
with open(filename, 'r') as input:
|
||||
for line in input:
|
||||
self.parse_header_line(line)
|
||||
|
||||
def add_test_case_line(self, function, argument):
|
||||
'''Parse a test case data line, looking for algorithm metadata tests.'''
|
||||
if function.endswith('_algorithm'):
|
||||
self.algorithms.add(argument)
|
||||
if function == 'hash_algorithm':
|
||||
self.hash_algorithms.add(argument)
|
||||
elif function == 'key_type':
|
||||
self.key_types.add(argument)
|
||||
elif function == 'ecc_key_types':
|
||||
self.ecc_curves.add(argument)
|
||||
|
||||
# Regex matching a *.data line containing a test function call and
|
||||
# its arguments. The actual definition is partly positional, but this
|
||||
# regex is good enough in practice.
|
||||
test_case_line_re = re.compile('(?!depends_on:)(\w+):([^\n :][^:\n]*)')
|
||||
def parse_test_cases(self, filename):
|
||||
'''Parse a test case file (*.data), looking for algorithm metadata tests.'''
|
||||
with open(filename, 'r') as input:
|
||||
for line in input:
|
||||
m = re.match(self.test_case_line_re, line)
|
||||
if m:
|
||||
self.add_test_case_line(m.group(1), m.group(2))
|
||||
|
||||
def gather_inputs(headers, test_suites):
|
||||
'''Read the list of inputs to test psa_constant_names with.'''
|
||||
inputs = Inputs()
|
||||
for header in headers:
|
||||
inputs.parse_header(header)
|
||||
for test_cases in test_suites:
|
||||
inputs.parse_test_cases(test_cases)
|
||||
inputs.gather_arguments()
|
||||
return inputs
|
||||
|
||||
def remove_file_if_exists(filename):
|
||||
'''Remove the specified file, ignoring errors.'''
|
||||
if not filename:
|
||||
return
|
||||
try:
|
||||
os.remove(filename)
|
||||
except:
|
||||
pass
|
||||
|
||||
def run_c(options, names):
|
||||
'''Generate and run a program to print out numerical values for names.'''
|
||||
c_name = None
|
||||
exe_name = None
|
||||
try:
|
||||
c_fd, c_name = tempfile.mkstemp(suffix='.c',
|
||||
dir='programs/psa')
|
||||
exe_suffix = '.exe' if platform.system() == 'Windows' else ''
|
||||
exe_name = c_name[:-2] + exe_suffix
|
||||
remove_file_if_exists(exe_name)
|
||||
c_file = os.fdopen(c_fd, 'w', encoding='ascii')
|
||||
c_file.write('''/* Generated by test_psa_constant_names.py */
|
||||
#include <stdio.h>
|
||||
#include <psa/crypto.h>
|
||||
int main(void)
|
||||
{
|
||||
''')
|
||||
for name in names:
|
||||
c_file.write(' printf("0x%08x\\n", {});\n'.format(name))
|
||||
c_file.write(''' return 0;
|
||||
}
|
||||
''')
|
||||
c_file.close()
|
||||
cc = os.getenv('CC', 'cc')
|
||||
subprocess.check_call([cc] +
|
||||
['-I' + dir for dir in options.include] +
|
||||
['-o', exe_name, c_name])
|
||||
os.remove(c_name)
|
||||
output = subprocess.check_output([exe_name])
|
||||
return output.decode('ascii').strip().split('\n')
|
||||
finally:
|
||||
remove_file_if_exists(exe_name)
|
||||
|
||||
normalize_strip_re = re.compile(r'\s+')
|
||||
def normalize(expr):
|
||||
'''Normalize the C expression so as not to care about trivial differences.
|
||||
Currently "trivial differences" means whitespace.'''
|
||||
expr = re.sub(normalize_strip_re, '', expr, len(expr))
|
||||
return expr.strip().split('\n')
|
||||
|
||||
def do_test(options, inputs, type, names):
|
||||
'''Test psa_constant_names for the specified type.
|
||||
Run program on names.
|
||||
Use inputs to figure out what arguments to pass to macros that take arguments.'''
|
||||
names = sorted(itertools.chain(*map(inputs.distribute_arguments, names)))
|
||||
values = run_c(options, names)
|
||||
output = subprocess.check_output([options.program, type] + values)
|
||||
outputs = output.decode('ascii').strip().split('\n')
|
||||
errors = [(type, name, value, output)
|
||||
for (name, value, output) in zip(names, values, outputs)
|
||||
if normalize(name) != normalize(output)]
|
||||
return len(names), errors
|
||||
|
||||
def report_errors(errors):
|
||||
'''Describe each case where the output is not as expected.'''
|
||||
for type, name, value, output in errors:
|
||||
print('For {} "{}", got "{}" (value: {})'
|
||||
.format(type, name, output, value))
|
||||
|
||||
def run_tests(options, inputs):
|
||||
'''Run psa_constant_names on all the gathered inputs.
|
||||
Return a tuple (count, errors) where count is the total number of inputs
|
||||
that were tested and errors is the list of cases where the output was
|
||||
not as expected.'''
|
||||
count = 0
|
||||
errors = []
|
||||
for type, names in [('status', inputs.statuses),
|
||||
('algorithm', inputs.algorithms),
|
||||
('ecc_curve', inputs.ecc_curves),
|
||||
('key_type', inputs.key_types),
|
||||
('key_usage', inputs.key_usage_flags)]:
|
||||
c, e = do_test(options, inputs, type, names)
|
||||
count += c
|
||||
errors += e
|
||||
return count, errors
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=globals()['__doc__'])
|
||||
parser.add_argument('--include', '-I',
|
||||
action='append', default=['include'],
|
||||
help='Directory for header files')
|
||||
parser.add_argument('--program',
|
||||
default='programs/psa/psa_constant_names',
|
||||
help='Program to test')
|
||||
options = parser.parse_args()
|
||||
headers = [os.path.join(options.include[0], 'psa/crypto.h')]
|
||||
test_suites = ['tests/suites/test_suite_psa_crypto_metadata.data']
|
||||
inputs = gather_inputs(headers, test_suites)
|
||||
count, errors = run_tests(options, inputs)
|
||||
report_errors(errors)
|
||||
if errors == []:
|
||||
print('{} test cases PASS'.format(count))
|
||||
else:
|
||||
print('{} test cases, {} FAIL'.format(count, len(errors)))
|
||||
exit(1)
|
||||
Loading…
Reference in a new issue