From 33406b645dea9a10a8ebf08e98b1c5d8face1b98 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Thu, 2 Nov 2023 18:48:39 +0100
Subject: [PATCH] Add a metatest program

This program can be used to validate that things that should be detected as
test failures are indeed caught, either by setting the test result to
MBEDTLS_TEST_RESULT_FAILED or by aborting the program.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 programs/.gitignore          |  1 +
 programs/Makefile            |  5 +++
 programs/test/CMakeLists.txt |  1 +
 programs/test/metatest.c     | 81 ++++++++++++++++++++++++++++++++++++
 4 files changed, 88 insertions(+)
 create mode 100644 programs/test/metatest.c

diff --git a/programs/.gitignore b/programs/.gitignore
index a641c31c4..2826a18a4 100644
--- a/programs/.gitignore
+++ b/programs/.gitignore
@@ -56,6 +56,7 @@ test/cpp_dummy_build
 test/cpp_dummy_build.cpp
 test/dlopen
 test/ecp-bench
+test/metatest
 test/query_compile_time_config
 test/query_included_headers
 test/selftest
diff --git a/programs/Makefile b/programs/Makefile
index 116883b83..7b538da4b 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -123,6 +123,7 @@ APPS = \
 	ssl/ssl_server \
 	ssl/ssl_server2 \
 	test/benchmark \
+	test/metatest \
 	test/query_compile_time_config \
 	test/query_included_headers \
 	test/selftest \
@@ -413,6 +414,10 @@ test/dlopen$(EXEXT): test/dlopen.c $(DEP)
 	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) test/dlopen.c $(LDFLAGS) $(DLOPEN_LDFLAGS) -o $@
 endif
 
+test/metatest$(EXEXT): test/metatest.c $(DEP)
+	echo "  CC    test/metatest.c"
+	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) test/metatest.c    $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@
+
 test/query_config.o: test/query_config.c test/query_config.h $(DEP)
 	echo "  CC    test/query_config.c"
 	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -c test/query_config.c -o $@
diff --git a/programs/test/CMakeLists.txt b/programs/test/CMakeLists.txt
index a75f8d923..8469d064d 100644
--- a/programs/test/CMakeLists.txt
+++ b/programs/test/CMakeLists.txt
@@ -3,6 +3,7 @@ set(libs
 )
 
 set(executables_libs
+    metatest
     query_included_headers
     selftest
     udp_proxy
diff --git a/programs/test/metatest.c b/programs/test/metatest.c
new file mode 100644
index 000000000..f03e5e703
--- /dev/null
+++ b/programs/test/metatest.c
@@ -0,0 +1,81 @@
+/** \file metatest.c
+ *
+ *  \brief Test features of the test framework.
+ */
+
+/*
+ *  Copyright The Mbed TLS Contributors
+ *  SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+ */
+
+#define MBEDTLS_ALLOW_PRIVATE_ACCESS
+
+#include <mbedtls/platform.h>
+#include "test/helpers.h"
+
+#include <stdio.h>
+#include <string.h>
+
+
+/****************************************************************/
+/* Command line entry point */
+/****************************************************************/
+
+typedef struct {
+    const char *name;
+    const char *platform;
+    void (*entry_point)(const char *name);
+} metatest_t;
+
+metatest_t metatests[] = {
+    { NULL, NULL, NULL }
+};
+
+static void help(FILE *out, const char *argv0)
+{
+    mbedtls_fprintf(out, "Usage: %s list|TEST\n", argv0);
+    mbedtls_fprintf(out, "Run a meta-test that should cause a test failure.\n");
+    mbedtls_fprintf(out, "With 'list', list the available tests and their platform requirement.\n");
+}
+
+int main(int argc, char *argv[])
+{
+    const char *argv0 = argc > 0 ? argv[0] : "metatest";
+    if (argc != 2) {
+        help(stderr, argv0);
+        mbedtls_exit(MBEDTLS_EXIT_FAILURE);
+    }
+
+    /* Support "-help", "--help", "--list", etc. */
+    const char *command = argv[1];
+    while (*command == '-') {
+        ++command;
+    }
+
+    if (strcmp(argv[1], "help") == 0) {
+        help(stdout, argv0);
+        mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
+    }
+    if (strcmp(argv[1], "list") == 0) {
+        for (const metatest_t *p = metatests; p->name != NULL; p++) {
+            mbedtls_printf("%s %s\n", p->name, p->platform);
+        }
+        mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
+    }
+
+    for (const metatest_t *p = metatests; p->name != NULL; p++) {
+        if (strcmp(argv[1], p->name) == 0) {
+            mbedtls_printf("Running metatest %s...\n", argv[1]);
+            p->entry_point(argv[1]);
+            mbedtls_printf("Running metatest %s... done, result=%d\n",
+                           argv[1], (int) mbedtls_test_info.result);
+            mbedtls_exit(mbedtls_test_info.result == MBEDTLS_TEST_RESULT_SUCCESS ?
+                         MBEDTLS_EXIT_SUCCESS :
+                         MBEDTLS_EXIT_FAILURE);
+        }
+    }
+
+    mbedtls_fprintf(stderr, "%s: FATAL: No such metatest: %s\n",
+                    argv0, command);
+    mbedtls_exit(MBEDTLS_EXIT_FAILURE);
+}