503 lines
16 KiB
C
503 lines
16 KiB
C
/*
|
|
* Armv8-A Cryptographic Extension support functions for Aarch64
|
|
*
|
|
* Copyright The Mbed TLS Contributors
|
|
* SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
|
|
*/
|
|
|
|
#if defined(__aarch64__) && !defined(__ARM_FEATURE_CRYPTO) && \
|
|
defined(__clang__) && __clang_major__ >= 4
|
|
/* TODO: Re-consider above after https://reviews.llvm.org/D131064 merged.
|
|
*
|
|
* The intrinsic declaration are guarded by predefined ACLE macros in clang:
|
|
* these are normally only enabled by the -march option on the command line.
|
|
* By defining the macros ourselves we gain access to those declarations without
|
|
* requiring -march on the command line.
|
|
*
|
|
* `arm_neon.h` could be included by any header file, so we put these defines
|
|
* at the top of this file, before any includes.
|
|
*/
|
|
#define __ARM_FEATURE_CRYPTO 1
|
|
/* See: https://arm-software.github.io/acle/main/acle.html#cryptographic-extensions
|
|
*
|
|
* `__ARM_FEATURE_CRYPTO` is deprecated, but we need to continue to specify it
|
|
* for older compilers.
|
|
*/
|
|
#define __ARM_FEATURE_AES 1
|
|
#define MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG
|
|
#endif
|
|
|
|
#include <string.h>
|
|
#include "common.h"
|
|
|
|
#if defined(MBEDTLS_AESCE_C)
|
|
|
|
#include "aesce.h"
|
|
|
|
#if defined(MBEDTLS_ARCH_IS_ARM64)
|
|
|
|
/* Compiler version checks. */
|
|
#if defined(__clang__)
|
|
# if __clang_major__ < 4
|
|
# error "Minimum version of Clang for MBEDTLS_AESCE_C is 4.0."
|
|
# endif
|
|
#elif defined(__GNUC__)
|
|
# if __GNUC__ < 6
|
|
# error "Minimum version of GCC for MBEDTLS_AESCE_C is 6.0."
|
|
# endif
|
|
#elif defined(_MSC_VER)
|
|
/* TODO: We haven't verified MSVC from 1920 to 1928. If someone verified that,
|
|
* please update this and document of `MBEDTLS_AESCE_C` in
|
|
* `mbedtls_config.h`. */
|
|
# if _MSC_VER < 1929
|
|
# error "Minimum version of MSVC for MBEDTLS_AESCE_C is 2019 version 16.11.2."
|
|
# endif
|
|
#endif
|
|
|
|
#ifdef __ARM_NEON
|
|
#include <arm_neon.h>
|
|
#else
|
|
#error "Target does not support NEON instructions"
|
|
#endif
|
|
|
|
#if !(defined(__ARM_FEATURE_CRYPTO) || defined(__ARM_FEATURE_AES)) || \
|
|
defined(MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG)
|
|
# if defined(__ARMCOMPILER_VERSION)
|
|
# if __ARMCOMPILER_VERSION <= 6090000
|
|
# error "Must use minimum -march=armv8-a+crypto for MBEDTLS_AESCE_C"
|
|
# else
|
|
# pragma clang attribute push (__attribute__((target("aes"))), apply_to=function)
|
|
# define MBEDTLS_POP_TARGET_PRAGMA
|
|
# endif
|
|
# elif defined(__clang__)
|
|
# pragma clang attribute push (__attribute__((target("aes"))), apply_to=function)
|
|
# define MBEDTLS_POP_TARGET_PRAGMA
|
|
# elif defined(__GNUC__)
|
|
# pragma GCC push_options
|
|
# pragma GCC target ("+crypto")
|
|
# define MBEDTLS_POP_TARGET_PRAGMA
|
|
# elif defined(_MSC_VER)
|
|
# error "Required feature(__ARM_FEATURE_AES) is not enabled."
|
|
# endif
|
|
#endif /* !(__ARM_FEATURE_CRYPTO || __ARM_FEATURE_AES) ||
|
|
MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG */
|
|
|
|
#if defined(__linux__) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
|
|
|
|
#include <asm/hwcap.h>
|
|
#include <sys/auxv.h>
|
|
|
|
signed char mbedtls_aesce_has_support_result = -1;
|
|
|
|
#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
|
|
/*
|
|
* AES instruction support detection routine
|
|
*/
|
|
int mbedtls_aesce_has_support_impl(void)
|
|
{
|
|
/* To avoid many calls to getauxval, cache the result. This is
|
|
* thread-safe, because we store the result in a char so cannot
|
|
* be vulnerable to non-atomic updates.
|
|
* It is possible that we could end up setting result more than
|
|
* once, but that is harmless.
|
|
*/
|
|
if (mbedtls_aesce_has_support_result == -1) {
|
|
unsigned long auxval = getauxval(AT_HWCAP);
|
|
if ((auxval & (HWCAP_ASIMD | HWCAP_AES)) ==
|
|
(HWCAP_ASIMD | HWCAP_AES)) {
|
|
mbedtls_aesce_has_support_result = 1;
|
|
} else {
|
|
mbedtls_aesce_has_support_result = 0;
|
|
}
|
|
}
|
|
return mbedtls_aesce_has_support_result;
|
|
}
|
|
#endif
|
|
|
|
#endif /* defined(__linux__) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY) */
|
|
|
|
/* Single round of AESCE encryption */
|
|
#define AESCE_ENCRYPT_ROUND \
|
|
block = vaeseq_u8(block, vld1q_u8(keys)); \
|
|
block = vaesmcq_u8(block); \
|
|
keys += 16
|
|
/* Two rounds of AESCE encryption */
|
|
#define AESCE_ENCRYPT_ROUND_X2 AESCE_ENCRYPT_ROUND; AESCE_ENCRYPT_ROUND
|
|
|
|
MBEDTLS_OPTIMIZE_FOR_PERFORMANCE
|
|
static uint8x16_t aesce_encrypt_block(uint8x16_t block,
|
|
unsigned char *keys,
|
|
int rounds)
|
|
{
|
|
/* 10, 12 or 14 rounds. Unroll loop. */
|
|
if (rounds == 10) {
|
|
goto rounds_10;
|
|
}
|
|
if (rounds == 12) {
|
|
goto rounds_12;
|
|
}
|
|
AESCE_ENCRYPT_ROUND_X2;
|
|
rounds_12:
|
|
AESCE_ENCRYPT_ROUND_X2;
|
|
rounds_10:
|
|
AESCE_ENCRYPT_ROUND_X2;
|
|
AESCE_ENCRYPT_ROUND_X2;
|
|
AESCE_ENCRYPT_ROUND_X2;
|
|
AESCE_ENCRYPT_ROUND_X2;
|
|
AESCE_ENCRYPT_ROUND;
|
|
|
|
/* AES AddRoundKey for the previous round.
|
|
* SubBytes, ShiftRows for the final round. */
|
|
block = vaeseq_u8(block, vld1q_u8(keys));
|
|
keys += 16;
|
|
|
|
/* Final round: no MixColumns */
|
|
|
|
/* Final AddRoundKey */
|
|
block = veorq_u8(block, vld1q_u8(keys));
|
|
|
|
return block;
|
|
}
|
|
|
|
/* Single round of AESCE decryption
|
|
*
|
|
* AES AddRoundKey, SubBytes, ShiftRows
|
|
*
|
|
* block = vaesdq_u8(block, vld1q_u8(keys));
|
|
*
|
|
* AES inverse MixColumns for the next round.
|
|
*
|
|
* This means that we switch the order of the inverse AddRoundKey and
|
|
* inverse MixColumns operations. We have to do this as AddRoundKey is
|
|
* done in an atomic instruction together with the inverses of SubBytes
|
|
* and ShiftRows.
|
|
*
|
|
* It works because MixColumns is a linear operation over GF(2^8) and
|
|
* AddRoundKey is an exclusive or, which is equivalent to addition over
|
|
* GF(2^8). (The inverse of MixColumns needs to be applied to the
|
|
* affected round keys separately which has been done when the
|
|
* decryption round keys were calculated.)
|
|
*
|
|
* block = vaesimcq_u8(block);
|
|
*/
|
|
#define AESCE_DECRYPT_ROUND \
|
|
block = vaesdq_u8(block, vld1q_u8(keys)); \
|
|
block = vaesimcq_u8(block); \
|
|
keys += 16
|
|
/* Two rounds of AESCE decryption */
|
|
#define AESCE_DECRYPT_ROUND_X2 AESCE_DECRYPT_ROUND; AESCE_DECRYPT_ROUND
|
|
|
|
static uint8x16_t aesce_decrypt_block(uint8x16_t block,
|
|
unsigned char *keys,
|
|
int rounds)
|
|
{
|
|
/* 10, 12 or 14 rounds. Unroll loop. */
|
|
if (rounds == 10) {
|
|
goto rounds_10;
|
|
}
|
|
if (rounds == 12) {
|
|
goto rounds_12;
|
|
}
|
|
AESCE_DECRYPT_ROUND_X2;
|
|
rounds_12:
|
|
AESCE_DECRYPT_ROUND_X2;
|
|
rounds_10:
|
|
AESCE_DECRYPT_ROUND_X2;
|
|
AESCE_DECRYPT_ROUND_X2;
|
|
AESCE_DECRYPT_ROUND_X2;
|
|
AESCE_DECRYPT_ROUND_X2;
|
|
AESCE_DECRYPT_ROUND;
|
|
|
|
/* The inverses of AES AddRoundKey, SubBytes, ShiftRows finishing up the
|
|
* last full round. */
|
|
block = vaesdq_u8(block, vld1q_u8(keys));
|
|
keys += 16;
|
|
|
|
/* Inverse AddRoundKey for inverting the initial round key addition. */
|
|
block = veorq_u8(block, vld1q_u8(keys));
|
|
|
|
return block;
|
|
}
|
|
|
|
/*
|
|
* AES-ECB block en(de)cryption
|
|
*/
|
|
int mbedtls_aesce_crypt_ecb(mbedtls_aes_context *ctx,
|
|
int mode,
|
|
const unsigned char input[16],
|
|
unsigned char output[16])
|
|
{
|
|
uint8x16_t block = vld1q_u8(&input[0]);
|
|
unsigned char *keys = (unsigned char *) (ctx->buf + ctx->rk_offset);
|
|
|
|
if (mode == MBEDTLS_AES_ENCRYPT) {
|
|
block = aesce_encrypt_block(block, keys, ctx->nr);
|
|
} else {
|
|
block = aesce_decrypt_block(block, keys, ctx->nr);
|
|
}
|
|
vst1q_u8(&output[0], block);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Compute decryption round keys from encryption round keys
|
|
*/
|
|
void mbedtls_aesce_inverse_key(unsigned char *invkey,
|
|
const unsigned char *fwdkey,
|
|
int nr)
|
|
{
|
|
int i, j;
|
|
j = nr;
|
|
vst1q_u8(invkey, vld1q_u8(fwdkey + j * 16));
|
|
for (i = 1, j--; j > 0; i++, j--) {
|
|
vst1q_u8(invkey + i * 16,
|
|
vaesimcq_u8(vld1q_u8(fwdkey + j * 16)));
|
|
}
|
|
vst1q_u8(invkey + i * 16, vld1q_u8(fwdkey + j * 16));
|
|
|
|
}
|
|
|
|
static inline uint32_t aes_rot_word(uint32_t word)
|
|
{
|
|
return (word << (32 - 8)) | (word >> 8);
|
|
}
|
|
|
|
static inline uint32_t aes_sub_word(uint32_t in)
|
|
{
|
|
uint8x16_t v = vreinterpretq_u8_u32(vdupq_n_u32(in));
|
|
uint8x16_t zero = vdupq_n_u8(0);
|
|
|
|
/* vaeseq_u8 does both SubBytes and ShiftRows. Taking the first row yields
|
|
* the correct result as ShiftRows doesn't change the first row. */
|
|
v = vaeseq_u8(zero, v);
|
|
return vgetq_lane_u32(vreinterpretq_u32_u8(v), 0);
|
|
}
|
|
|
|
/*
|
|
* Key expansion function
|
|
*/
|
|
static void aesce_setkey_enc(unsigned char *rk,
|
|
const unsigned char *key,
|
|
const size_t key_bit_length)
|
|
{
|
|
static uint8_t const rcon[] = { 0x01, 0x02, 0x04, 0x08, 0x10,
|
|
0x20, 0x40, 0x80, 0x1b, 0x36 };
|
|
/* See https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.197.pdf
|
|
* - Section 5, Nr = Nk + 6
|
|
* - Section 5.2, the length of round keys is Nb*(Nr+1)
|
|
*/
|
|
const uint32_t key_len_in_words = key_bit_length / 32; /* Nk */
|
|
const size_t round_key_len_in_words = 4; /* Nb */
|
|
const size_t rounds_needed = key_len_in_words + 6; /* Nr */
|
|
const size_t round_keys_len_in_words =
|
|
round_key_len_in_words * (rounds_needed + 1); /* Nb*(Nr+1) */
|
|
const uint32_t *rko_end = (uint32_t *) rk + round_keys_len_in_words;
|
|
|
|
memcpy(rk, key, key_len_in_words * 4);
|
|
|
|
for (uint32_t *rki = (uint32_t *) rk;
|
|
rki + key_len_in_words < rko_end;
|
|
rki += key_len_in_words) {
|
|
|
|
size_t iteration = (size_t) (rki - (uint32_t *) rk) / key_len_in_words;
|
|
uint32_t *rko;
|
|
rko = rki + key_len_in_words;
|
|
rko[0] = aes_rot_word(aes_sub_word(rki[key_len_in_words - 1]));
|
|
rko[0] ^= rcon[iteration] ^ rki[0];
|
|
rko[1] = rko[0] ^ rki[1];
|
|
rko[2] = rko[1] ^ rki[2];
|
|
rko[3] = rko[2] ^ rki[3];
|
|
if (rko + key_len_in_words > rko_end) {
|
|
/* Do not write overflow words.*/
|
|
continue;
|
|
}
|
|
#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
|
|
switch (key_bit_length) {
|
|
case 128:
|
|
break;
|
|
case 192:
|
|
rko[4] = rko[3] ^ rki[4];
|
|
rko[5] = rko[4] ^ rki[5];
|
|
break;
|
|
case 256:
|
|
rko[4] = aes_sub_word(rko[3]) ^ rki[4];
|
|
rko[5] = rko[4] ^ rki[5];
|
|
rko[6] = rko[5] ^ rki[6];
|
|
rko[7] = rko[6] ^ rki[7];
|
|
break;
|
|
}
|
|
#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Key expansion, wrapper
|
|
*/
|
|
int mbedtls_aesce_setkey_enc(unsigned char *rk,
|
|
const unsigned char *key,
|
|
size_t bits)
|
|
{
|
|
switch (bits) {
|
|
case 128:
|
|
case 192:
|
|
case 256:
|
|
aesce_setkey_enc(rk, key, bits);
|
|
break;
|
|
default:
|
|
return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#if defined(MBEDTLS_GCM_C)
|
|
|
|
#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 5
|
|
/* Some intrinsics are not available for GCC 5.X. */
|
|
#define vreinterpretq_p64_u8(a) ((poly64x2_t) a)
|
|
#define vreinterpretq_u8_p128(a) ((uint8x16_t) a)
|
|
static inline poly64_t vget_low_p64(poly64x2_t __a)
|
|
{
|
|
uint64x2_t tmp = (uint64x2_t) (__a);
|
|
uint64x1_t lo = vcreate_u64(vgetq_lane_u64(tmp, 0));
|
|
return (poly64_t) (lo);
|
|
}
|
|
#endif /* !__clang__ && __GNUC__ && __GNUC__ == 5*/
|
|
|
|
/* vmull_p64/vmull_high_p64 wrappers.
|
|
*
|
|
* Older compilers miss some intrinsic functions for `poly*_t`. We use
|
|
* uint8x16_t and uint8x16x3_t as input/output parameters.
|
|
*/
|
|
#if defined(__GNUC__) && !defined(__clang__)
|
|
/* GCC reports incompatible type error without cast. GCC think poly64_t and
|
|
* poly64x1_t are different, that is different with MSVC and Clang. */
|
|
#define MBEDTLS_VMULL_P64(a, b) vmull_p64((poly64_t) a, (poly64_t) b)
|
|
#else
|
|
/* MSVC reports `error C2440: 'type cast'` with cast. Clang does not report
|
|
* error with/without cast. And I think poly64_t and poly64x1_t are same, no
|
|
* cast for clang also. */
|
|
#define MBEDTLS_VMULL_P64(a, b) vmull_p64(a, b)
|
|
#endif
|
|
static inline uint8x16_t pmull_low(uint8x16_t a, uint8x16_t b)
|
|
{
|
|
|
|
return vreinterpretq_u8_p128(
|
|
MBEDTLS_VMULL_P64(
|
|
vget_low_p64(vreinterpretq_p64_u8(a)),
|
|
vget_low_p64(vreinterpretq_p64_u8(b))
|
|
));
|
|
}
|
|
|
|
static inline uint8x16_t pmull_high(uint8x16_t a, uint8x16_t b)
|
|
{
|
|
return vreinterpretq_u8_p128(
|
|
vmull_high_p64(vreinterpretq_p64_u8(a),
|
|
vreinterpretq_p64_u8(b)));
|
|
}
|
|
|
|
/* GHASH does 128b polynomial multiplication on block in GF(2^128) defined by
|
|
* `x^128 + x^7 + x^2 + x + 1`.
|
|
*
|
|
* Arm64 only has 64b->128b polynomial multipliers, we need to do 4 64b
|
|
* multiplies to generate a 128b.
|
|
*
|
|
* `poly_mult_128` executes polynomial multiplication and outputs 256b that
|
|
* represented by 3 128b due to code size optimization.
|
|
*
|
|
* Output layout:
|
|
* | | | |
|
|
* |------------|-------------|-------------|
|
|
* | ret.val[0] | h3:h2:00:00 | high 128b |
|
|
* | ret.val[1] | :m2:m1:00 | middle 128b |
|
|
* | ret.val[2] | : :l1:l0 | low 128b |
|
|
*/
|
|
static inline uint8x16x3_t poly_mult_128(uint8x16_t a, uint8x16_t b)
|
|
{
|
|
uint8x16x3_t ret;
|
|
uint8x16_t h, m, l; /* retval high/middle/low */
|
|
uint8x16_t c, d, e;
|
|
|
|
h = pmull_high(a, b); /* h3:h2:00:00 = a1*b1 */
|
|
l = pmull_low(a, b); /* : :l1:l0 = a0*b0 */
|
|
c = vextq_u8(b, b, 8); /* :c1:c0 = b0:b1 */
|
|
d = pmull_high(a, c); /* :d2:d1:00 = a1*b0 */
|
|
e = pmull_low(a, c); /* :e2:e1:00 = a0*b1 */
|
|
m = veorq_u8(d, e); /* :m2:m1:00 = d + e */
|
|
|
|
ret.val[0] = h;
|
|
ret.val[1] = m;
|
|
ret.val[2] = l;
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Modulo reduction.
|
|
*
|
|
* See: https://www.researchgate.net/publication/285612706_Implementing_GCM_on_ARMv8
|
|
*
|
|
* Section 4.3
|
|
*
|
|
* Modular reduction is slightly more complex. Write the GCM modulus as f(z) =
|
|
* z^128 +r(z), where r(z) = z^7+z^2+z+ 1. The well known approach is to
|
|
* consider that z^128 ≡r(z) (mod z^128 +r(z)), allowing us to write the 256-bit
|
|
* operand to be reduced as a(z) = h(z)z^128 +l(z)≡h(z)r(z) + l(z). That is, we
|
|
* simply multiply the higher part of the operand by r(z) and add it to l(z). If
|
|
* the result is still larger than 128 bits, we reduce again.
|
|
*/
|
|
static inline uint8x16_t poly_mult_reduce(uint8x16x3_t input)
|
|
{
|
|
uint8x16_t const ZERO = vdupq_n_u8(0);
|
|
|
|
uint64x2_t r = vreinterpretq_u64_u8(vdupq_n_u8(0x87));
|
|
#if defined(__GNUC__)
|
|
/* use 'asm' as an optimisation barrier to prevent loading MODULO from
|
|
* memory. It is for GNUC compatible compilers.
|
|
*/
|
|
asm ("" : "+w" (r));
|
|
#endif
|
|
uint8x16_t const MODULO = vreinterpretq_u8_u64(vshrq_n_u64(r, 64 - 8));
|
|
uint8x16_t h, m, l; /* input high/middle/low 128b */
|
|
uint8x16_t c, d, e, f, g, n, o;
|
|
h = input.val[0]; /* h3:h2:00:00 */
|
|
m = input.val[1]; /* :m2:m1:00 */
|
|
l = input.val[2]; /* : :l1:l0 */
|
|
c = pmull_high(h, MODULO); /* :c2:c1:00 = reduction of h3 */
|
|
d = pmull_low(h, MODULO); /* : :d1:d0 = reduction of h2 */
|
|
e = veorq_u8(c, m); /* :e2:e1:00 = m2:m1:00 + c2:c1:00 */
|
|
f = pmull_high(e, MODULO); /* : :f1:f0 = reduction of e2 */
|
|
g = vextq_u8(ZERO, e, 8); /* : :g1:00 = e1:00 */
|
|
n = veorq_u8(d, l); /* : :n1:n0 = d1:d0 + l1:l0 */
|
|
o = veorq_u8(n, f); /* o1:o0 = f1:f0 + n1:n0 */
|
|
return veorq_u8(o, g); /* = o1:o0 + g1:00 */
|
|
}
|
|
|
|
/*
|
|
* GCM multiplication: c = a times b in GF(2^128)
|
|
*/
|
|
void mbedtls_aesce_gcm_mult(unsigned char c[16],
|
|
const unsigned char a[16],
|
|
const unsigned char b[16])
|
|
{
|
|
uint8x16_t va, vb, vc;
|
|
va = vrbitq_u8(vld1q_u8(&a[0]));
|
|
vb = vrbitq_u8(vld1q_u8(&b[0]));
|
|
vc = vrbitq_u8(poly_mult_reduce(poly_mult_128(va, vb)));
|
|
vst1q_u8(&c[0], vc);
|
|
}
|
|
|
|
#endif /* MBEDTLS_GCM_C */
|
|
|
|
#if defined(MBEDTLS_POP_TARGET_PRAGMA)
|
|
#if defined(__clang__)
|
|
#pragma clang attribute pop
|
|
#elif defined(__GNUC__)
|
|
#pragma GCC pop_options
|
|
#endif
|
|
#undef MBEDTLS_POP_TARGET_PRAGMA
|
|
#endif
|
|
|
|
#endif /* MBEDTLS_ARCH_IS_ARM64 */
|
|
|
|
#endif /* MBEDTLS_AESCE_C */
|